Support fitting lenses on any dataset and equal-parts mixes
The fit corpus was limited to three hardcoded choices (wikitext, Semantic-Harmless, mixed), and any other id was rejected. Now any HuggingFace dataset id works, and any number of them can be ticked to fit on an equal-parts mix, shuffled. n_prompts now counts training SEQUENCES (what the fit iterates over) instead of source rows: each dataset is packed up to its quota, so the number entered is exactly what runs, regardless of the dataset. The fixed dropdown becomes a checkable dataset library persisted in localStorage. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
daeb127651
commit
d9773394e3
+2
-2
@@ -175,7 +175,7 @@ class FitRequest(BaseModel):
|
||||
dtype: str = "bf16"
|
||||
quant: str | None = None
|
||||
n_prompts: int = 100
|
||||
dataset: str = fitting.DATASET_WIKITEXT # or DATASET_HARMLESS, or "mixed"
|
||||
datasets: list[str] = [fitting.DATASET_WIKITEXT] # any HF ids; several = equal-parts mix
|
||||
devices: list[str] = ["cuda:0"]
|
||||
name: str | None = None
|
||||
dim_batch: int | None = None
|
||||
@@ -811,7 +811,7 @@ def api_fit(req: FitRequest):
|
||||
n_prompts=req.n_prompts,
|
||||
dtype=req.dtype,
|
||||
quant=req.quant,
|
||||
dataset=req.dataset,
|
||||
datasets=req.datasets,
|
||||
devices=req.devices,
|
||||
name=req.name,
|
||||
dim_batch=req.dim_batch,
|
||||
|
||||
Reference in New Issue
Block a user