Cleanup

Author: nielstron

eval/README.md

@@ -32,15 +32,15 @@ eval/
 For the MRI models, we provide an inference harness for the C++ HumanEval multi-region dataset.
 To execute task 11 on the 1-region dataset with constraints and traces enabled, use the following command:
 ```bash
-python3 -m constrained_diffusion.eval.mri.generic_inference
-  --max-tokens 256
-  --model_name deepseek-ai/deepseek-coder-6.7b-base
-  --seed 0
-  --temp 1
-  --dataset-name HumanEval/MRI/cpp/1
-  --constrained True
-  --trace True
-  --task_id /11_
+python3 -m constrained_diffusion.eval.mri.generic_inference \
+  --max-tokens 256 \
+  --model_name deepseek-ai/deepseek-coder-6.7b-base \
+  --seed 0 \
+  --temp 1 \
+  --dataset-name HumanEval/MRI/cpp/1 \
+  --constrained True \
+  --trace True \
+  --task_id /11_ 
 ```
 
 For the diffusion LLMs, use the following command for the SMILES dataset.

eval/dllm/run_dllm.py

@@ -20,27 +20,21 @@ def compute_needed_gpus(size_model, size_gpu):
 
 subsets = [
     "THUDM/humaneval-x/cpp",
-    "jsonschema_r",
-    # "nmuendler/json-mode-eval-cleaned",
+    "jsonschema",
     "smiles",
-    # "jsonschema",
-    # "sed",
 ]
 temps = ["0.2"]
 gap_sizes = {
-    "jsonschema_r": [0],  # , 1, 5],
     "jsonschema": [0],  # , 1, 5],
     "THUDM/humaneval-x/cpp": [0],  # , 1, 2],
-    "nmuendler/json-mode-eval-cleaned": [0],  # , 1, 5],
     "smiles": [0],  # , 1, 5],
-    "sed": [0],
 }
 step_sizes = [32]
 seeds = [0, 1, 2, 3]
 configs = [
     ("", "_synth"),
 ]
-constraineds = [True]
+constraineds = [True, False]
 
 
 def find_available_gpus(gpus, n):
@@ -133,7 +127,7 @@ def find_available_gpus(gpus, n):
     else:
         suffix = "nc"
     command = (
-        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_outoforder.eval.dllm.generic_inference "
+        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_diffusion.eval.dllm.generic_inference "
         f"--max-tokens 256 --model_name {model} --seed {seed} --temp {temp} --trace False  --dataset-name '{subset}' --inject_gap_size {gap_size} --max_total_injections {gap_size} --steps {num_step} "
         f"--constrained {constrained} --output_file 'results/{subset.replace('/', '_')}_{model.replace('/', '_')}_s={seed}_t={temp}_gs={gap_size}_sz={num_step}{name}_{suffix}.jsonl' {config}"
     )

eval/dllm/run_dllm_step_ablation.py

@@ -16,26 +16,22 @@ def compute_needed_gpus(size_model, size_gpu):
 
 
 subsets = [
-    # "nmuendler/json-mode-eval-cleaned",
     "smiles",
     "THUDM/humaneval-x/cpp",
-    "jsonschema_r",
-    # "sed",
+    "jsonschema",
 ]
 temps = ["0.2"]
 gap_sizes = {
-    "jsonschema_r": [0],  # , 1, 5],
+    "jsonschema": [0],  # , 1, 5],
     "THUDM/humaneval-x/cpp": [0],  # , 1, 2],
-    "nmuendler/json-mode-eval-cleaned": [0],  # , 1, 5],
     "smiles": [0],  # , 1, 5],
-    "sed": [0],
 }
 step_sizes = [16, 64, 128, 256]
 seeds = [0, 1, 2, 3]
 configs = [
     ("", "_synth"),
 ]
-constraineds = [True]
+constraineds = [True, False]
 
 
 def find_available_gpus(gpus, n):
@@ -128,7 +124,7 @@ def find_available_gpus(gpus, n):
     else:
         suffix = "nc"
     command = (
-        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_outoforder.eval.dllm.generic_inference "
+        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_diffusion.eval.dllm.generic_inference "
         f"--max-tokens 256 --model_name {model} --seed {seed} --temp {temp} --trace False  --dataset-name '{subset}' --inject_gap_size {gap_size} --max_total_injections {gap_size} --steps {num_step} "
         f"--constrained {constrained} --output_file 'results/{subset.replace('/', '_')}_{model.replace('/', '_')}_s={seed}_t={temp}_gs={gap_size}_sz={num_step}{name}_{suffix}.jsonl' {config}"
     )

eval/mri/run_fim.py

@@ -42,7 +42,7 @@ def compute_needed_gpus(size_model, size_gpu):
 configs = [
     ("", "_synth"),
 ]
-constraineds = [True]
+constraineds = [True, False]
 
 
 def find_available_gpus(gpus, n):
@@ -132,7 +132,7 @@ def find_available_gpus(gpus, n):
     else:
         suffix = "nc"
     command = (
-        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_outoforder.eval.mri.generic_inference "
+        f"PYTHONPATH=. CUDA_VISIBLE_DEVICES={','.join(str(i) for i in cuda_devices)} python3 -m constrained_diffusion.eval.mri.generic_inference "
         f"--max-tokens 256 --model_name {model} --seed {seed} --temp {temp} --trace False  --dataset-name '{subset}' --inject_gap_size {gap_size} --max_total_injections {gap_size} "
         f"--constrained {constrained} --output_file 'results/{subset.replace('/', '_')}_{model.replace('/', '_')}_s={seed}_t={temp}_gs={gap_size}{name}_{suffix}.jsonl' {config}"
     )

example.py

@@ -142,7 +142,7 @@ def main():
     print(prompt)
     print("Took {:.2f} seconds to generate.".format(time_taken_autocompletion))
     print("----------- Code ------------------")
-    print(autocompletion)
+    print(autocompletion or extracted)
 
 
 if __name__ == "__main__":