Create README.md

README.md

@@ -0,0 +1,47 @@
+```python
+from transformers import  RobertaTokenizerFast, AutoModelForSequenceClassification
+from datasets  import load_dataset, Dataset
+from functools import partial
+from tqdm.auto import tqdm
+tqdm._instances.clear()
+
+def tokenize_function(example):
+    inputs = tokenizer(
+        example["sentence"],
+        example["context"],
+        max_length=512,
+        truncation=True,
+        padding="max_length",
+    )
+    return inputs
+
+def create_windowed_context_ds(context_l, example, idx):
+    example["context"] = context_l[idx]
+    return example
+
+def create_windowed_context(raw_dataset, window_size):
+    df_pandas = raw_dataset['train'].to_pandas()
+    len1 = len(raw_dataset['train'])
+    context_l = []
+    for i in tqdm(range(len1)):
+        if i - window_size <0:
+            context_l.append(' '.join(df_pandas['sentence'][0:window_size]))
+        else:
+            if i + window_size > len1 :
+                context_l.append(' '.join(df_pandas['sentence'][i - window_size:-1]))
+            else:
+                context_l.append(' '.join(df_pandas['sentence'][i - window_size:i + window_size]))
+    return context_l
+
+model = AutoModelForSequenceClassification.from_pretrained('HeTree/HeConEspc', num_labels=2)
+tokenizer = RobertaTokenizerFast.from_pretrained('HeTree/HeConEspc')
+raw_dataset = load_dataset('HeTree/MevakerConcSen')
+window_size = 5
+context_l = create_windowed_context(raw_dataset, window_size)
+raw_dataset_window = raw_dataset.map(partial(create_windowed_context_ds, context_l), batched=False, with_indices=True)
+tokenized_data = raw_dataset_window.map(tokenize_function, batched=True)
+
+
+
+
+```