Merge pull request #315 from ntumlgroup/dropout

Eleven1Liu · web-flow · commit cb651c1fe36e · 2023-07-24T22:59:56.000+08:00
Homogenizing Dropout
diff --git a/docs/examples/plot_KimCNN_quickstart.py b/docs/examples/plot_KimCNN_quickstart.py
@@ -45,7 +45,7 @@
 # We consider the following settings for the KimCNN model.
 
 model_name = "KimCNN"
-network_config = {"embed_dropout": 0.2, "encoder_dropout": 0.2, "filter_sizes": [2, 4, 8], "num_filter_per_size": 128}
+network_config = {"embed_dropout": 0.2, "post_encoder_dropout": 0.2, "filter_sizes": [2, 4, 8], "num_filter_per_size": 128}
 learning_rate = 0.0003
 model = init_model(
     model_name=model_name,
diff --git a/docs/examples/plot_bert_quickstart.py b/docs/examples/plot_bert_quickstart.py
@@ -54,7 +54,7 @@
 
 model_name = "BERT"
 network_config = {
-    "dropout": 0.1,
+    "encoder_hidden_dropout": 0.1,
     "lm_weight": "bert-base-uncased",
 }
 learning_rate = 0.00003
diff --git a/example_config/EUR-Lex-57k/bert.yml b/example_config/EUR-Lex-57k/bert.yml
@@ -27,7 +27,7 @@ val_metric: RP@5
 model_name: BERT
 init_weight: null
 network_config:
-  dropout: 0.1
+  encoder_hidden_dropout: 0.1
   lm_weight: bert-base-uncased
   lm_window: 512
 
diff --git a/example_config/EUR-Lex-57k/bert_lwan.yml b/example_config/EUR-Lex-57k/bert_lwan.yml
@@ -26,7 +26,7 @@ val_metric: RP@5
 model_name: BERTAttention
 init_weight: null
 network_config:
-  dropout: 0.1
+  post_encoder_dropout: 0.1
   lm_weight: bert-base-uncased
   lm_window: 512
   attention_type: singlehead
diff --git a/example_config/EUR-Lex-57k/bert_lwan_tune.yml b/example_config/EUR-Lex-57k/bert_lwan_tune.yml
@@ -31,7 +31,7 @@ model_name: BERTAttention
 loss_function: binary_cross_entropy_with_logits
 init_weight: null
 network_config:
-  dropout: ['grid_search', [0, 0.1, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.1, 0.2, 0.4]]
   lm_weight: bert-base-uncased
   lm_window: 512
   attention_type: singlehead
diff --git a/example_config/EUR-Lex-57k/bigru_lwan.yml b/example_config/EUR-Lex-57k/bigru_lwan.yml
@@ -29,7 +29,7 @@ model_name: BiGRULWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   rnn_dim: 256
   rnn_layers: 1
 
diff --git a/example_config/EUR-Lex-57k/bigru_lwan_tune.yml b/example_config/EUR-Lex-57k/bigru_lwan_tune.yml
@@ -34,7 +34,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   rnn_dim: ['grid_search', [256, 512, 1024]]
   rnn_layers: 1
 
diff --git a/example_config/EUR-Lex-57k/cnn_lwan.yml b/example_config/EUR-Lex-57k/cnn_lwan.yml
@@ -29,7 +29,7 @@ model_name: CNNLWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.2
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   filter_sizes: [8]
   num_filter_per_size: 256
 
diff --git a/example_config/EUR-Lex-57k/cnn_lwan_tune.yml b/example_config/EUR-Lex-57k/cnn_lwan_tune.yml
@@ -35,7 +35,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: tanh
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [32, 64, 128, 256, 512, 1024]]
 
diff --git a/example_config/EUR-Lex-57k/kim_cnn.yml b/example_config/EUR-Lex-57k/kim_cnn.yml
@@ -29,7 +29,7 @@ model_name: KimCNN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   filter_sizes: [8]
   num_filter_per_size: 1024
 
diff --git a/example_config/EUR-Lex-57k/kim_cnn_tune.yml b/example_config/EUR-Lex-57k/kim_cnn_tune.yml
@@ -35,7 +35,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: relu
   embed_dropout: ['grid_search', [0, 0.2, 0.4]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [128, 256, 384, 512, 768, 1024]]
 
diff --git a/example_config/EUR-Lex/bigru_lwan.yml b/example_config/EUR-Lex/bigru_lwan.yml
@@ -25,7 +25,7 @@ model_name: BiGRULWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   rnn_dim: 512
   rnn_layers: 1
 
diff --git a/example_config/EUR-Lex/bigru_lwan_tune.yml b/example_config/EUR-Lex/bigru_lwan_tune.yml
@@ -31,7 +31,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   rnn_dim: ['grid_search', [256, 512, 1024]]
   rnn_layers: 1
 
diff --git a/example_config/EUR-Lex/cnn_lwan.yml b/example_config/EUR-Lex/cnn_lwan.yml
@@ -25,7 +25,7 @@ model_name: CNNLWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   filter_sizes: [8]
   num_filter_per_size: 128
 
diff --git a/example_config/EUR-Lex/cnn_lwan_tune.yml b/example_config/EUR-Lex/cnn_lwan_tune.yml
@@ -32,7 +32,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: tanh
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [32, 64, 128, 256, 512, 1024]]
 
diff --git a/example_config/EUR-Lex/kim_cnn.yml b/example_config/EUR-Lex/kim_cnn.yml
@@ -25,7 +25,7 @@ model_name: KimCNN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.2
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   filter_sizes: [8]
   num_filter_per_size: 1024
 
diff --git a/example_config/EUR-Lex/kim_cnn_tune.yml b/example_config/EUR-Lex/kim_cnn_tune.yml
@@ -32,7 +32,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: relu
   embed_dropout: ['grid_search', [0, 0.2, 0.4]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [128, 256, 384, 512, 768, 1024]]
 
diff --git a/example_config/EUR-Lex/xml_cnn.yml b/example_config/EUR-Lex/xml_cnn.yml
@@ -24,7 +24,7 @@ val_metric: P@1
 model_name: XMLCNN
 network_config:
   embed_dropout: 0.2
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   filter_sizes: [2, 4, 8]
   hidden_dim: 512
   num_filter_per_size: 256  # filter channels
diff --git a/example_config/MIMIC-50/bert_lwan.yml b/example_config/MIMIC-50/bert_lwan.yml
@@ -32,7 +32,7 @@ val_metric: P@5
 model_name: BERTAttention
 init_weight: null
 network_config:
-  dropout: 0.2
+  post_encoder_dropout: 0.2
   lm_weight: emilyalsentzer/Bio_ClinicalBERT
   lm_window: 512
   num_heads: 8
diff --git a/example_config/MIMIC-50/bigru_lwan.yml b/example_config/MIMIC-50/bigru_lwan.yml
@@ -30,7 +30,7 @@ model_name: BiGRULWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.8
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   rnn_dim: 1024
   rnn_layers: 1
 
diff --git a/example_config/MIMIC-50/bigru_lwan_tune.yml b/example_config/MIMIC-50/bigru_lwan_tune.yml
@@ -34,7 +34,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: ['grid_search', ['kaiming_uniform', 'xavier_uniform']]
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
   rnn_dim: ['grid_search', [256, 512, 1024, 2048]]
   rnn_layers: ['grid_search', [1, 2]]
 
diff --git a/example_config/MIMIC-50/caml.yml b/example_config/MIMIC-50/caml.yml
@@ -31,7 +31,7 @@ val_metric: P@5
 model_name: CAML
 init_weight: null
 network_config:
-  dropout: 0.2
+  embed_dropout: 0.2
   filter_sizes: [10]
   num_filter_per_size: 50
 
diff --git a/example_config/MIMIC-50/caml_tune.yml b/example_config/MIMIC-50/caml_tune.yml
@@ -35,7 +35,7 @@ init_weight: null
 network_config:
   num_filter_per_size: ['grid_search', [50, 150, 250, 350, 450, 550]]
   filter_sizes: ['grid_search', [[2], [4], [6], [8], [10]]]
-  dropout: ['grid_search', [0.2, 0.4, 0.6, 0.8]]
+  embed_dropout: ['grid_search', [0.2, 0.4, 0.6, 0.8]]
 
 # pretrained vocab / embeddings
 vocab_file: data/MIMIC-50/vocab.csv
diff --git a/example_config/MIMIC-50/cnn.yml b/example_config/MIMIC-50/cnn.yml
@@ -31,7 +31,7 @@ init_weight: xavier_uniform
 network_config:
   activation: tanh
   embed_dropout: 0.2
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   filter_sizes: [4]
   num_filter_per_size: 500
 
diff --git a/example_config/MIMIC/bigru_lwan.yml b/example_config/MIMIC/bigru_lwan.yml
@@ -30,7 +30,7 @@ model_name: BiGRULWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   rnn_dim: 1024
   rnn_layers: 1
 
diff --git a/example_config/MIMIC/bigru_lwan_tune.yml b/example_config/MIMIC/bigru_lwan_tune.yml
@@ -35,7 +35,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   rnn_dim: ['grid_search', [256, 512, 1024]]
   rnn_layers: 1
 
diff --git a/example_config/MIMIC/cnn_lwan.yml b/example_config/MIMIC/cnn_lwan.yml
@@ -30,7 +30,7 @@ model_name: CNNLWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0
+  post_encoder_dropout: 0
   filter_sizes: [10]
   num_filter_per_size: 256
 
diff --git a/example_config/MIMIC/cnn_lwan_tune.yml b/example_config/MIMIC/cnn_lwan_tune.yml
@@ -35,7 +35,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [10]
   num_filter_per_size: ['grid_search', [32, 64, 128, 256, 512, 1024]]
 
diff --git a/example_config/MIMIC/kim_cnn.yml b/example_config/MIMIC/kim_cnn.yml
@@ -30,7 +30,7 @@ model_name: KimCNN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0
+  post_encoder_dropout: 0
   filter_sizes: [10]
   num_filter_per_size: 768
 
diff --git a/example_config/MIMIC/kim_cnn_tune.yml b/example_config/MIMIC/kim_cnn_tune.yml
@@ -36,7 +36,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: relu
   embed_dropout: ['grid_search', [0, 0.2, 0.4]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [10]
   num_filter_per_size: ['grid_search', [128, 256, 384, 512, 768, 1024]]
 
diff --git a/example_config/Wiki10-31K/bigru_lwan.yml b/example_config/Wiki10-31K/bigru_lwan.yml
@@ -25,7 +25,7 @@ model_name: BiGRULWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.4
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   rnn_dim: 512
   rnn_layers: 1
 
diff --git a/example_config/Wiki10-31K/bigru_lwan_tune.yml b/example_config/Wiki10-31K/bigru_lwan_tune.yml
@@ -32,7 +32,7 @@ loss_function: binary_cross_entropy_with_logits
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   rnn_dim: ['grid_search', [256, 512, 1024]]
   rnn_layers: 1
 
diff --git a/example_config/Wiki10-31K/cnn_lwan.yml b/example_config/Wiki10-31K/cnn_lwan.yml
@@ -25,7 +25,7 @@ model_name: CNNLWAN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   filter_sizes: [8]
   num_filter_per_size: 128
 
diff --git a/example_config/Wiki10-31K/cnn_lwan_tune.yml b/example_config/Wiki10-31K/cnn_lwan_tune.yml
@@ -33,7 +33,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: tanh
   embed_dropout: ['grid_search', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [32, 64, 128, 256, 512, 1024]]
 
diff --git a/example_config/Wiki10-31K/kim_cnn.yml b/example_config/Wiki10-31K/kim_cnn.yml
@@ -25,7 +25,7 @@ model_name: KimCNN
 init_weight: kaiming_uniform
 network_config:
   embed_dropout: 0.2
-  encoder_dropout: 0.4
+  post_encoder_dropout: 0.4
   filter_sizes: [8]
   num_filter_per_size: 768
 
diff --git a/example_config/Wiki10-31K/kim_cnn_tune.yml b/example_config/Wiki10-31K/kim_cnn_tune.yml
@@ -33,7 +33,7 @@ init_weight: kaiming_uniform
 network_config:
   activation: relu
   embed_dropout: ['grid_search', [0, 0.2, 0.4]]
-  encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
+  post_encoder_dropout: ['grid_search', [0, 0.2, 0.4]]
   filter_sizes: [8]
   num_filter_per_size: ['grid_search', [128, 256, 384, 512, 768, 1024]]
 
diff --git a/example_config/rcv1/cnn_tune.yml b/example_config/rcv1/cnn_tune.yml
@@ -33,7 +33,7 @@ init_weight: kaiming_uniform
 network_config:
   filter_sizes: ['grid_search', [[2,4,8], [4,6]]]
   embed_dropout: ['choice', [0, 0.2, 0.4, 0.6, 0.8]]
-  encoder_dropout: ['choice', [0, 0.2, 0.4, 0.6, 0.8]]
+  post_encoder_dropout: ['choice', [0, 0.2, 0.4, 0.6, 0.8]]
   num_filter_per_size: 128 # filter channels
   activation: relu
 
diff --git a/example_config/rcv1/kim_cnn.yml b/example_config/rcv1/kim_cnn.yml
@@ -24,7 +24,7 @@ val_metric: Macro-F1
 model_name: KimCNN
 network_config:
   embed_dropout: 0.2
-  encoder_dropout: 0.2
+  post_encoder_dropout: 0.2
   filter_sizes: [2, 4, 8]
   num_filter_per_size: 128 # filter channels
 
diff --git a/example_config/rcv1/xml_cnn.yml b/example_config/rcv1/xml_cnn.yml
@@ -24,7 +24,7 @@ val_metric: P@1
 model_name: XMLCNN
 network_config:
   embed_dropout: 0.5
-  encoder_dropout: 0.5
+  post_encoder_dropout: 0.5
   filter_sizes: [2, 4, 8]
   hidden_dim: 512
   num_filter_per_size: 128 # filter channels
diff --git a/libmultilabel/nn/networks/bert.py b/libmultilabel/nn/networks/bert.py
@@ -7,18 +7,34 @@ class BERT(nn.Module):
 
     Args:
         num_classes (int): Total number of classes.
-        dropout (float): The dropout rate of the word embedding. Defaults to 0.2.
+        encoder_hidden_dropout (float): The dropout rate of the feed forward sublayer in each BERT layer. Defaults to 0.1.
+        encoder_attention_dropout (float): The dropout rate of the attention sublayer in each BERT layer. Defaults to 0.1.
+        post_encoder_dropout (float): The dropout rate of the dropout layer after the BERT model. Defaults to 0.
         lm_weight (str): Pretrained model name or path. Defaults to 'bert-base-cased'.
         lm_window (int): Length of the subsequences to be split before feeding them to
             the language model. Defaults to 512.
     """
 
-    def __init__(self, num_classes, dropout=0.2, lm_weight="bert-base-cased", lm_window=512, **kwargs):
+    def __init__(
+        self,
+        num_classes,
+        encoder_hidden_dropout=0.1,
+        encoder_attention_dropout=0.1,
+        post_encoder_dropout=0,
+        lm_weight="bert-base-cased",
+        lm_window=512,
+        **kwargs,
+    ):
         super().__init__()
         self.lm_window = lm_window
 
         self.lm = AutoModelForSequenceClassification.from_pretrained(
-            lm_weight, num_labels=num_classes, hidden_dropout_prob=dropout, torchscript=True
+            lm_weight,
+            num_labels=num_classes,
+            hidden_dropout_prob=encoder_hidden_dropout,
+            attention_probs_dropout_prob=encoder_attention_dropout,
+            classifier_dropout=post_encoder_dropout,
+            torchscript=True,
         )
 
     def forward(self, input):
diff --git a/libmultilabel/nn/networks/bert_attention.py b/libmultilabel/nn/networks/bert_attention.py
diff --git a/libmultilabel/nn/networks/caml.py b/libmultilabel/nn/networks/caml.py
diff --git a/libmultilabel/nn/networks/kim_cnn.py b/libmultilabel/nn/networks/kim_cnn.py
diff --git a/libmultilabel/nn/networks/labelwise_attention_networks.py b/libmultilabel/nn/networks/labelwise_attention_networks.py
diff --git a/libmultilabel/nn/networks/modules.py b/libmultilabel/nn/networks/modules.py
diff --git a/libmultilabel/nn/networks/xml_cnn.py b/libmultilabel/nn/networks/xml_cnn.py

Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@`
`54`	`54`
`55`	`55`	`model_name = "BERT"`
`56`	`56`	`network_config = {`
`57`		`- "dropout": 0.1,`
	`57`	`+ "encoder_hidden_dropout": 0.1,`
`58`	`58`	`"lm_weight": "bert-base-uncased",`
`59`	`59`	`}`
`60`	`60`	`learning_rate = 0.00003`