diff --git a/.gitignore b/.gitignore index 463236b..d22cb1c 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,5 @@ logs/* .ruff_cache/ engibench_studies/* +workshops/dcc26/artifacts/* +workshops/dcc26/optional_artifacts/* diff --git a/README.md b/README.md index 6b828b5..0a793fc 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,21 @@ We have some colab notebooks that show how to use some of the EngiBench/EngiOpt * [Example easy model (GAN)](https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/main/example_easy_model.ipynb) * [Example hard model (Diffusion)](https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/main/example_hard_model.ipynb) +## Workshop notebooks + +For the DCC'26 hands-on tutorial flow, see: + +- `workshops/dcc26/participant/00_setup_api_warmup.ipynb` +- `workshops/dcc26/participant/01_train_generate.ipynb` +- `workshops/dcc26/participant/02_evaluate_metrics.ipynb` +- `workshops/dcc26/participant/03_add_new_problem_scaffold.ipynb` + +Facilitator solutions are in: + +- `workshops/dcc26/solutions/` + +See `workshops/dcc26/README.md` for agenda mapping, fallback path, and artifact outputs. + ## Citing diff --git a/pyproject.toml b/pyproject.toml index 36881c4..f267845 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,10 @@ target-version = "py39" ######################################## LINTING ######################################## [tool.ruff.lint] select = ["ALL"] +exclude = [ + "workshops/dcc26/**/*.ipynb", + "workshops/dcc26/utils/**/*.py", +] ignore = [ "ANN", # flake8-annotations (mypy's job) "COM812", # missing-trailing-comma (conflicts with formatter) diff --git a/workshops/dcc26/NOTEBOOK_PEDAGOGY_BLUEPRINT.md b/workshops/dcc26/NOTEBOOK_PEDAGOGY_BLUEPRINT.md new file mode 100644 index 0000000..19d6299 --- /dev/null +++ b/workshops/dcc26/NOTEBOOK_PEDAGOGY_BLUEPRINT.md @@ -0,0 +1,131 @@ +# DCC26 Notebook Pedagogy Blueprint (Pre-write Source) + +This document is the canonical pre-write for workshop notebooks. Notebooks should be generated from this structure, not authored directly as raw `.ipynb` first. + +## Teaching Design Principles + +1. Every technical step is paired with a markdown teaching cell. +2. Every code section has local context: why, inputs, outputs, checks, failure modes. +3. Benchmark science is explicit: objective, feasibility, diversity, novelty, reproducibility. +4. Discussion prompts are embedded and mapped to workshop breakout questions. +5. Participant and solution tracks share the same pedagogical arc; only implementation detail differs. + +## Common Cell Pattern + +For each section: + +- Purpose: why this step matters for benchmark credibility +- Inputs: what artifacts/variables are required +- Action: code operation performed +- Success check: what output indicates correctness +- Failure modes: common pitfalls and fixes +- Discussion bridge: one reflection question + +--- + +## Notebook 00: Setup + API Warmup + +### Learning objective +Understand EngiBench benchmark contract components and reproducibility controls. + +### Section plan +1. Read-me-first + copy mode + runtime expectation +2. Concept cell: EngiBench vs model libraries +3. Environment bootstrap +4. Reproducibility cell (seed, versions) +5. Problem instantiation (`Beams2D`) + inspection +6. Dataset inspection and shape sanity +7. Render one sample and explain representation +8. Explicit constraint violation check with interpretation +9. Reflection prompts tied to comparability across papers + +### Discussion trigger +Which benchmark settings must be fixed for fair method comparison? + +--- + +## Notebook 01: Train + Generate + +### Learning objective +Implement an EngiOpt model against EngiBench data while preserving evaluation-ready artifacts. + +### Section plan +1. Read-me-first + copy mode + expected runtime +2. Concept cell: inverse design framing, conditional generation assumptions +3. Bootstrap deps and imports +4. Configuration and artifact contract +5. Data subset construction and rationale (runtime vs fidelity) +6. Model definition and optimizer +7. Training loop with diagnostics + expected loss behavior +8. Generation from test conditions +9. Quick feasibility precheck (not final evaluation) +10. Artifact export contract (npy/json/checkpoint/history/curve) +11. Optional W&B logging: train curve, scalar logs, artifact bundle +12. Visual sanity grid +13. Discussion prompt: training loss vs engineering validity mismatch + +### Discussion trigger +Can lower train reconstruction loss worsen simulator objective or feasibility? + +--- + +## Notebook 02: Evaluate + Metrics + +### Learning objective +Run robust benchmark evaluation and interpret trade-offs beyond objective score. + +### Section plan +1. Read-me-first + copy mode + expected runtime +2. Concept cell: why objective-only reporting is incomplete +3. Bootstrap deps and imports +4. Artifact loading strategy (local -> optional W&B -> local auto-build) +5. Per-sample evaluation loop (constraint + simulate) +6. Metric layer: + - objective means and gap + - improvement rate + - feasibility/violation rates + - diversity proxy + - novelty-to-train proxy +7. Export layer: CSV + histogram + scatter + grid +8. Optional W&B evaluation logging (table + images + summary) +9. Interpretation rubric with examples +10. Breakout prompts mapped to workshop proposal + +### Discussion trigger +Which missing metric would change conclusions for your domain? + +--- + +## Notebook 03: Add New Problem Scaffold + +### Learning objective +Understand minimal interface required for a reusable EngiBench-style benchmark problem. + +### Section plan +1. Read-me-first + copy mode +2. Concept cell: benchmark-ready problem checklist +3. Scaffold imports and abstract contract explanation +4. Minimal `Problem` implementation skeleton +5. Toy simulator and constraints +6. Registration/discovery and deterministic behavior +7. Contribution checklist for real domains +8. Reflection prompts on leakage, units, and reproducibility metadata + +### Discussion trigger +What metadata is minimally required so another lab can reproduce your new benchmark? + +--- + +## Participant vs Solution Policy + +- Participant notebooks: keep code TODOs, but each TODO has explicit completion checks and expected outputs. +- Solution notebooks: complete implementations plus concise inline comments for non-obvious logic only. +- Both tracks: keep identical markdown structure for pedagogical alignment. + +## Quality Gate Before Publishing + +1. All code cells compile. +2. Solution Notebook 01+02 execute end-to-end in workshop env. +3. Artifact contract is consistent between Notebook 01 and 02. +4. Copy-safe links use `#copy=true`. +5. Standalone readability check: each notebook understandable without live lecture. diff --git a/workshops/dcc26/README.md b/workshops/dcc26/README.md new file mode 100644 index 0000000..245f9f4 --- /dev/null +++ b/workshops/dcc26/README.md @@ -0,0 +1,104 @@ +# DCC 2026 Workshop Notebook Suite + +This folder contains the DCC'26 hands-on notebook suite for benchmarking AI methods in engineering design with EngiBench and EngiOpt. + +It is split into two tracks: + +- `participant/`: notebooks with guided `PUBLIC FILL-IN` cells for attendees +- `solutions/`: fully completed facilitator notebooks + +## Workshop flow (3.5h) + +- `participant/00_setup_api_warmup.ipynb` and `solutions/00_setup_api_warmup.ipynb` (10-15 min) + - Environment setup + - Problem + dataset inspection + - Rendering and constraint checks + +- `participant/01_train_generate.ipynb` and `solutions/01_train_generate.ipynb` (30 min) + - Lightweight training using `engiopt.cgan_2d.Generator` + - Deterministic seeds + - Artifact export for downstream evaluation (runtime/W&B optional transport) + +- `participant/02_evaluate_metrics.ipynb` and `solutions/02_evaluate_metrics.ipynb` (20 min) + - Constraint validation + - Physics simulation + - Baseline comparison + - Metric and artifact export + +- `participant/03_add_new_problem_scaffold.ipynb` and `solutions/03_add_new_problem_scaffold.ipynb` (25 min) + - Ambitious `Problem` scaffold (`PlanarManipulatorCoDesignProblem`, not currently in EngiBench) + - PyBullet-based robotics co-design simulation and optimization loop + - Mapping to contribution docs + +## Runtime assumptions + +- Primary live problem: `Beams2D` +- No container-dependent problems are required during workshop exercises +- W&B integration is optional and disabled by default + +## Colab setup + +Use `requirements-colab.txt` only as a local convenience snapshot. +The notebook bootstrap cells are the runtime source of truth for Colab. + +All notebooks now include a conditional dependency bootstrap cell: + +- On Colab: installs required packages automatically. +- On local envs: skips install by default (`FORCE_INSTALL = False`). +- Note: notebooks that use EngiOpt install it from the EngiOpt GitHub branch bootstrap. + +## Open in Colab + +Use these `?copy=true` links for workshop sharing so attendees are prompted to create their own Drive copy first. + +- Participant 00: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/participant/00_setup_api_warmup.ipynb?copy=true +- Participant 01: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/participant/01_train_generate.ipynb?copy=true +- Participant 02: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/participant/02_evaluate_metrics.ipynb?copy=true +- Participant 03: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/participant/03_add_new_problem_scaffold.ipynb?copy=true +- Solution 00: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/solutions/00_setup_api_warmup.ipynb?copy=true +- Solution 01: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/solutions/01_train_generate.ipynb?copy=true +- Solution 02: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/solutions/02_evaluate_metrics.ipynb?copy=true +- Solution 03: https://colab.research.google.com/github/IDEALLab/EngiOpt/blob/codex/dcc26-workshop-notebooks/workshops/dcc26/solutions/03_add_new_problem_scaffold.ipynb?copy=true + +## Output artifacts + +By default, solution notebooks write generated artifacts to: + +- Local/Jupyter: `workshops/dcc26/artifacts/` +- Google Colab runtime: `/content/dcc26_artifacts/` (no Google Drive permission needed) + +Optional: + +- You can enable W&B artifact upload/download in Notebook 01/02 by setting `USE_WANDB_ARTIFACTS = True`. +- Notebook 01 logs training dynamics (`train/loss`) and can upload checkpoint/history/plots as artifact payload. +- Notebook 02 can log evaluation metrics, tables, and figures to W&B. +- W&B is disabled by default so participants can run without account setup or API keys. +- Notebook 02 auto-builds Notebook 01-style artifacts locally with EngiOpt if they are missing (`AUTO_BUILD_ARTIFACTS_IF_MISSING = True`). + +These include: + +- `generated_designs.npy` +- `baseline_designs.npy` +- `conditions.json` +- `engiopt_cgan2d_generator_supervised.pt` +- `training_history.csv` +- `training_curve.png` +- `metrics_summary.csv` +- `objective_histogram.png` +- `objective_scatter.png` +- `design_grid.png` + +## Facilitator fallback policy + +If runtime is constrained: + +1. Skip Notebook 01 and run `02_evaluate_metrics.ipynb`; it can build required artifacts automatically. +2. Or reuse a previously saved checkpoint/artifact set from W&B or local runtime files. +3. Keep `03_add_new_problem_scaffold.ipynb` as the capstone for extensibility. + +## Suggested pre-workshop checks + +1. Run all notebooks once in fresh Colab runtime. +2. Confirm dataset download succeeds. +3. Confirm artifacts are generated in the expected folder. +4. Confirm no cell requires W&B auth unless explicitly enabled. diff --git a/workshops/dcc26/participant/00_setup_api_warmup.ipynb b/workshops/dcc26/participant/00_setup_api_warmup.ipynb new file mode 100644 index 0000000..db000d8 --- /dev/null +++ b/workshops/dcc26/participant/00_setup_api_warmup.ipynb @@ -0,0 +1,379 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e84ac3df", + "metadata": {}, + "source": [ + "# Notebook 00 (Participant): Setup + API Warmup\n", + "\n", + "This chapter introduces the benchmark interface itself.\n", + "Your target is to read a problem definition as a reproducible scientific object, not just an API object.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d6bb2d73", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "a74f46db", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n", + "\n", + "### Public exercise legend\n", + "- `PUBLIC FILL-IN CELL`: this is the part you edit during the workshop.\n", + "- `CHECKPOINT`: run immediately after your edits; if it fails, fix before moving on.\n", + "- `IF YOU ARE STUCK`: use the hint comments in that same cell (do not jump ahead).\n" + ] + }, + { + "cell_type": "markdown", + "id": "e71be25d", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "Learning goals:\n", + "- identify what EngiBench fixes (problem contract),\n", + "- identify what researchers can vary (methods),\n", + "- inspect conditions, objectives, and constraints with reproducibility in mind.\n" + ] + }, + { + "cell_type": "markdown", + "id": "16df002f", + "metadata": {}, + "source": [ + "## Why this warmup matters\n", + "\n", + "In engineering-design ML, many apparent gains come from hidden evaluation differences.\n", + "This warmup is about controlling that risk: understanding exactly what is held constant by the benchmark.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d483dfbe", + "metadata": {}, + "source": [ + "## Optional install cell (fresh Colab)\n", + "\n", + "Run this cell only on a fresh Colab runtime or if imports fail.\n", + "Local environments can usually skip it.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de02c488", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"matplotlib\", \"seaborn\"]\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "35d3b7b8", + "metadata": {}, + "source": [ + "### Step 1 - Initialize reproducible session\n", + "\n", + "Set seed and print versions.\n", + "If this step is inconsistent across machines, downstream comparisons are not interpretable.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbdaa6d8", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import engibench\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "SEED = 7\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)\n", + "\n", + "print(\"engibench version:\", engibench.__version__)\n", + "print(\"seed:\", SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "2e7a064e", + "metadata": {}, + "source": [ + "### Step 2 - Instantiate the benchmark problem (PUBLIC FILL-IN)\n", + "\n", + "Create `Beams2D` and inspect its key fields.\n", + "\n", + "What this step teaches:\n", + "- how a benchmark problem defines design space + objectives + conditions,\n", + "- which fields are part of the public contract you must preserve for fair comparison.\n", + "\n", + "Success criteria:\n", + "- you can name each contract field and explain its role.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ab92d5b", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 00-A\n", + "# Goal: instantiate the benchmark problem and inspect the full contract.\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "problem = None # Example: Beams2D(seed=SEED)\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "if problem is None:\n", + " raise RuntimeError(\"Set `problem` before running this cell (example: Beams2D(seed=SEED)).\")\n", + "\n", + "print(\"Problem class:\", type(problem).__name__)\n", + "print(\"Design space:\", problem.design_space)\n", + "print(\"Objectives:\", problem.objectives)\n", + "print(\"Conditions instance:\", problem.conditions)\n", + "print(\"Condition keys:\", problem.conditions_keys)\n", + "print(\"Dataset ID:\", problem.dataset_id)\n", + "\n", + "# CHECKPOINT\n", + "assert hasattr(problem, \"design_space\"), \"Problem is missing design_space\"\n", + "assert hasattr(problem, \"objectives\"), \"Problem is missing objectives\"\n", + "assert len(problem.conditions_keys) > 0, \"conditions_keys should not be empty\"\n", + "print(\"Checkpoint passed: problem contract is visible and ready.\")" + ] + }, + { + "cell_type": "markdown", + "id": "2d0d6faf", + "metadata": {}, + "source": [ + "### Step 3 - Inspect dataset structure (PUBLIC FILL-IN)\n", + "\n", + "Load one train sample and inspect keys + shapes.\n", + "\n", + "What this step teaches:\n", + "- how conditions and designs are stored in the benchmark dataset,\n", + "- what must be serialized later when handing off artifacts between notebooks.\n", + "\n", + "Success criteria:\n", + "- `design` has the expected spatial shape,\n", + "- `config` contains exactly the condition keys used by the problem.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "851ef517", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 00-B\n", + "# Goal: inspect one training sample and build a valid config dictionary.\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "dataset = None # Example: problem.dataset\n", + "sample_idx = 0\n", + "# design = ... # np.array from dataset['train']['optimal_design'][sample_idx]\n", + "# config = ... # dict over problem.conditions_keys\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "if dataset is None:\n", + " raise RuntimeError(\"Set `dataset = problem.dataset` before running.\")\n", + "if \"design\" not in locals() or \"config\" not in locals():\n", + " raise RuntimeError(\"Define both `design` and `config` in the START FILL section.\")\n", + "\n", + "print(dataset)\n", + "print(\"sample_idx:\", sample_idx)\n", + "print(\"design shape:\", np.array(design).shape)\n", + "print(\"config:\", config)\n", + "\n", + "# CHECKPOINT\n", + "assert tuple(np.array(design).shape) == tuple(problem.design_space.shape), (\n", + " f\"design shape mismatch: expected {problem.design_space.shape}, got {np.array(design).shape}\"\n", + ")\n", + "missing = [k for k in problem.conditions_keys if k not in config]\n", + "assert not missing, f\"config missing condition keys: {missing}\"\n", + "print(\"Checkpoint passed: dataset sample + config are valid.\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3334a35", + "metadata": {}, + "source": [ + "### Step 4 - Visualize one benchmark design\n", + "\n", + "Render a sample and interpret what visual features correspond to feasible structure.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34f0fb89", + "metadata": {}, + "outputs": [], + "source": [ + "# Render the sampled design (run after TODO 3)\n", + "fig, ax = problem.render(design)\n", + "ax.set_title(\"Participant: sampled Beams2D design\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e7201d50", + "metadata": {}, + "source": [ + "### Step 5 - Test constraint semantics (PUBLIC FILL-IN)\n", + "\n", + "Run a deliberate mismatch case.\n", + "\n", + "Why this matters:\n", + "- robust benchmarking requires transparent failure modes,\n", + "- you should be able to explain *why* a design/config pair is invalid.\n", + "\n", + "Success criteria:\n", + "- you can trigger and inspect at least one violation message.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20936577", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 00-C\n", + "# Goal: force a constraint mismatch and inspect violation diagnostics.\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "bad_config = dict(config)\n", + "# bad_config['volfrac'] = ...\n", + "# violations = ...\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "if \"violations\" not in locals():\n", + " raise RuntimeError(\"Define `violations` in the START FILL section.\")\n", + "\n", + "print(\"Violation count:\", len(violations))\n", + "if violations:\n", + " for i, v in enumerate(violations[:5]):\n", + " print(f\" [{i}]\", v)\n", + "else:\n", + " print(\"No violations found. Try a more aggressive volfrac mismatch.\")\n", + "\n", + "# CHECKPOINT\n", + "assert hasattr(violations, \"__len__\"), \"violations should be a sized collection\"\n", + "print(\"Checkpoint passed: constraint semantics inspected.\")" + ] + }, + { + "cell_type": "markdown", + "id": "9b75437c", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "bbfd552e", + "metadata": {}, + "source": [ + "## Next\n", + "\n", + "Proceed to Notebook 01 to connect this benchmark interface to a concrete generative model pipeline.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f41cb555", + "metadata": {}, + "source": [ + "## Reflection prompts\n", + "\n", + "- Which benchmark fields must be reported in every paper for fair comparison?\n", + "- Which hidden defaults are most likely to create accidental unfairness?\n" + ] + }, + { + "cell_type": "markdown", + "id": "08c2b28a", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/participant/01_train_generate.ipynb b/workshops/dcc26/participant/01_train_generate.ipynb new file mode 100644 index 0000000..2af18bd --- /dev/null +++ b/workshops/dcc26/participant/01_train_generate.ipynb @@ -0,0 +1,504 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0f062bbd", + "metadata": {}, + "source": [ + "# Notebook 01 (Participant): Train + Generate with EngiOpt CGAN-2D\n", + "\n", + "You will implement the full train-and-generate path and produce artifacts consumed by Notebook 02.\n" + ] + }, + { + "cell_type": "markdown", + "id": "51af8729", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "71b11843", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n", + "\n", + "### Public exercise legend\n", + "- `PUBLIC FILL-IN CELL`: edit this cell directly.\n", + "- `CHECKPOINT`: run and verify before continuing.\n", + "- `IF YOU ARE STUCK`: use hint comments in the same cell.\n" + ] + }, + { + "cell_type": "markdown", + "id": "dcc7d40a", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "This chapter is about **method integration under benchmark constraints**.\n", + "Success means reproducible artifacts and interpretable diagnostics, not only low training loss.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "edfc8c25", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"sqlitedict\", \"matplotlib\", \"tqdm\", \"tyro\", \"wandb\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "353f81db", + "metadata": {}, + "source": [ + "## Part A: Setup\n", + "\n", + "Lock down runtime, seeds, and artifact paths before writing model logic.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ed1e478f", + "metadata": {}, + "source": [ + "### EngiBench vs EngiOpt roles in this notebook\n", + "\n", + "- EngiBench: defines data semantics, constraints, and simulator objective.\n", + "- EngiOpt: defines the generative model family and training dynamics.\n", + "\n", + "Keep this separation explicit in your reasoning and reporting.\n" + ] + }, + { + "cell_type": "markdown", + "id": "c9b50673", + "metadata": {}, + "source": [ + "### Step 1 - Configure reproducible environment\n", + "\n", + "Set all global controls once; downstream cells should rely on these values only.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2540a3a6", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import random\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "try:\n", + " from engiopt.cgan_2d.cgan_2d import Generator as EngiOptCGAN2DGenerator\n", + "except ModuleNotFoundError as exc:\n", + " raise ModuleNotFoundError(\n", + " \"Could not import engiopt model class. Run the bootstrap cell first; on Colab, restart runtime after install if needed.\"\n", + " ) from exc\n", + "\n", + "USE_WANDB_ARTIFACTS = False\n", + "WANDB_PROJECT = \"dcc26-workshop\"\n", + "WANDB_ENTITY = None\n", + "WANDB_ARTIFACT_NAME = \"dcc26_beams2d_generated_artifacts\"\n", + "WANDB_ARTIFACT_ALIAS = \"latest\"\n", + "WANDB_LOG_TRAINING = True\n", + "\n", + "\n", + "def resolve_artifact_dir(create: bool = False) -> Path:\n", + " in_colab = \"google.colab\" in sys.modules\n", + " path = Path(\"/content/dcc26_artifacts\") if in_colab else Path(\"workshops/dcc26/artifacts\")\n", + " if create:\n", + " path.mkdir(parents=True, exist_ok=True)\n", + " return path\n", + "\n", + "\n", + "SEED = 7\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)\n", + "th.manual_seed(SEED)\n", + "if th.cuda.is_available():\n", + " th.cuda.manual_seed_all(SEED)\n", + "\n", + "DEVICE = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + "print(\"device:\", DEVICE)\n", + "\n", + "ARTIFACT_DIR = resolve_artifact_dir(create=True)\n", + "print(\"artifact dir:\", ARTIFACT_DIR)\n", + "\n", + "CKPT_PATH = ARTIFACT_DIR / \"engiopt_cgan2d_generator_supervised.pt\"\n", + "HISTORY_PATH = ARTIFACT_DIR / \"training_history.csv\"\n", + "TRAIN_CURVE_PATH = ARTIFACT_DIR / \"training_curve.png\"\n", + "LATENT_DIM = 32" + ] + }, + { + "cell_type": "markdown", + "id": "0998508c", + "metadata": {}, + "source": [ + "### Step 2 - Build training slice from EngiBench dataset\n", + "\n", + "Use a compact subset for workshop runtime; treat this as a pedagogical approximation, not final benchmark protocol.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5047da37", + "metadata": {}, + "outputs": [], + "source": [ + "problem = Beams2D(seed=SEED)\n", + "train_ds = problem.dataset[\"train\"]\n", + "test_ds = problem.dataset[\"test\"]\n", + "\n", + "condition_keys = problem.conditions_keys\n", + "print(\"condition keys:\", condition_keys)\n", + "\n", + "N_TRAIN = 512\n", + "subset_idx = np.random.default_rng(SEED).choice(len(train_ds), size=N_TRAIN, replace=False)\n", + "\n", + "conds_np = np.stack([np.array(train_ds[k])[subset_idx].astype(np.float32) for k in condition_keys], axis=1)\n", + "designs_np = np.array(train_ds[\"optimal_design\"])[subset_idx].astype(np.float32)\n", + "targets_np = (designs_np * 2.0) - 1.0\n", + "\n", + "print(\"conditions shape:\", conds_np.shape)\n", + "print(\"designs shape:\", designs_np.shape)\n", + "print(\"target range:\", float(targets_np.min()), \"to\", float(targets_np.max()))" + ] + }, + { + "cell_type": "markdown", + "id": "5da6a98e", + "metadata": {}, + "source": [ + "### Step 3 - Implement model setup (PUBLIC FILL-IN)\n", + "\n", + "Instantiate generator, optimizer, and loss exactly once.\n", + "\n", + "Success criteria:\n", + "- noise tensor shape is `(batch, LATENT_DIM)`,\n", + "- model output shape matches `problem.design_space.shape`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01999c27", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 01-A\n", + "# Goal: set up the EngiOpt generator + training primitives.\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "model = None\n", + "optimizer = None\n", + "criterion = None\n", + "\n", + "\n", + "def sample_noise(batch_size: int) -> th.Tensor:\n", + " # Return standard normal latent vectors on DEVICE.\n", + " raise NotImplementedError(\"Implement sample_noise\")\n", + "\n", + "\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "if model is None or optimizer is None or criterion is None:\n", + " raise RuntimeError(\"Define model, optimizer, and criterion in the START FILL block.\")\n", + "\n", + "# CHECKPOINT: validate latent shape and forward-pass shape\n", + "z_probe = sample_noise(4)\n", + "assert tuple(z_probe.shape) == (4, LATENT_DIM), f\"Expected (4, {LATENT_DIM}), got {tuple(z_probe.shape)}\"\n", + "cond_probe = th.tensor(conds_np[:4], dtype=th.float32, device=DEVICE)\n", + "with th.no_grad():\n", + " pred_probe = model(z_probe, cond_probe)\n", + "assert tuple(pred_probe.shape[1:]) == tuple(problem.design_space.shape), (\n", + " f\"Output shape mismatch: expected tail {problem.design_space.shape}, got {tuple(pred_probe.shape[1:])}\"\n", + ")\n", + "print(\"Checkpoint passed: model setup is consistent with problem representation.\")" + ] + }, + { + "cell_type": "markdown", + "id": "78d28002", + "metadata": {}, + "source": [ + "### Step 4 - Implement train/load logic (PUBLIC FILL-IN)\n", + "\n", + "Track loss per epoch and persist checkpoint/history outputs.\n", + "\n", + "Success criteria:\n", + "- training path writes checkpoint + history + curve,\n", + "- load path restores a checkpoint without retraining.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c1b69b4", + "metadata": {}, + "outputs": [], + "source": [ + "TRAIN_FROM_SCRATCH = True\n", + "EPOCHS = 8\n", + "BATCH_SIZE = 64\n", + "\n", + "# PUBLIC FILL-IN CELL 01-B\n", + "# Goal: train quickly for workshop runtime or load an existing checkpoint.\n", + "\n", + "train_losses = []\n", + "\n", + "if TRAIN_FROM_SCRATCH:\n", + " # START FILL -----------------------------------------------------------\n", + " # 1) Build DataLoader from conds_np + targets_np\n", + " # 2) Run epoch loop with model.train()\n", + " # 3) For each batch: predict, compute loss, backward, optimizer step\n", + " # 4) Append epoch-average loss to train_losses\n", + " # 5) Save checkpoint to CKPT_PATH\n", + " # 6) Save history CSV to HISTORY_PATH\n", + " # 7) Save training curve figure to TRAIN_CURVE_PATH\n", + " raise NotImplementedError(\"Implement TRAIN_FROM_SCRATCH branch\")\n", + " # END FILL -------------------------------------------------------------\n", + "elif CKPT_PATH.exists():\n", + " # START FILL -----------------------------------------------------------\n", + " # Load checkpoint into model and, if available, load history CSV.\n", + " raise NotImplementedError(\"Implement checkpoint load branch\")\n", + " # END FILL -------------------------------------------------------------\n", + "else:\n", + " raise FileNotFoundError(f\"Checkpoint not found at {CKPT_PATH}. Train first or provide checkpoint.\")\n", + "\n", + "# CHECKPOINT\n", + "assert CKPT_PATH.exists(), f\"Missing checkpoint: {CKPT_PATH}\"\n", + "assert HISTORY_PATH.exists(), f\"Missing history CSV: {HISTORY_PATH}\"\n", + "print(\"Checkpoint passed: train/load artifacts are ready for generation step.\")" + ] + }, + { + "cell_type": "markdown", + "id": "9c006d13", + "metadata": {}, + "source": [ + "### Step 5 - Implement generation logic (PUBLIC FILL-IN)\n", + "\n", + "Generate conditioned designs on held-out test conditions.\n", + "\n", + "Success criteria:\n", + "- generated and baseline arrays are shape-compatible,\n", + "- condition records are JSON-serializable and aligned with samples.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c36231fe", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 01-C\n", + "# Goal: create generated designs + baseline designs + condition records.\n", + "\n", + "N_SAMPLES = 24\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "# Suggested sequence:\n", + "# 1) sample indices from test_ds\n", + "# 2) build test_conds and baseline_designs\n", + "# 3) run model in eval/no_grad with sample_noise\n", + "# 4) map tanh output [-1,1] -> [0,1] and clip\n", + "# 5) build `conditions_records` as list[dict]\n", + "raise NotImplementedError(\"Implement generation block\")\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "# CHECKPOINT\n", + "assert \"gen_designs\" in locals(), \"Define gen_designs\"\n", + "assert \"baseline_designs\" in locals(), \"Define baseline_designs\"\n", + "assert \"test_conds\" in locals(), \"Define test_conds\"\n", + "assert \"conditions_records\" in locals(), \"Define conditions_records\"\n", + "assert gen_designs.shape == baseline_designs.shape, \"Generated and baseline shapes must match\"\n", + "assert len(conditions_records) == gen_designs.shape[0], \"conditions_records length mismatch\"\n", + "print(\"Checkpoint passed: generation outputs are valid and aligned.\")" + ] + }, + { + "cell_type": "markdown", + "id": "d0020828", + "metadata": {}, + "source": [ + "### Step 6 - Implement artifact export (PUBLIC FILL-IN)\n", + "\n", + "Notebook 02 expects these files as a strict handoff contract.\n", + "Treat artifact naming and file format as part of benchmark reproducibility.\n", + "\n", + "Required files:\n", + "- `generated_designs.npy`\n", + "- `baseline_designs.npy`\n", + "- `conditions.json`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c16e7fbd", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 01-D\n", + "# Goal: export Notebook 02 handoff artifacts (plus optional extras).\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "# Required exports:\n", + "# - np.save(ARTIFACT_DIR / 'generated_designs.npy', gen_designs)\n", + "# - np.save(ARTIFACT_DIR / 'baseline_designs.npy', baseline_designs)\n", + "# - json dump of conditions_records -> conditions.json\n", + "# Recommended exports:\n", + "# - checkpoint (.pt), training_history.csv, training_curve.png\n", + "raise NotImplementedError(\"Implement artifact export block\")\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "# CHECKPOINT\n", + "required_files = [\n", + " ARTIFACT_DIR / \"generated_designs.npy\",\n", + " ARTIFACT_DIR / \"baseline_designs.npy\",\n", + " ARTIFACT_DIR / \"conditions.json\",\n", + "]\n", + "missing = [str(f) for f in required_files if not f.exists()]\n", + "if missing:\n", + " raise RuntimeError(\"Missing required artifacts:\\\\n\" + \"\\\\n\".join(missing))\n", + "print(\"Checkpoint passed: Notebook 02 handoff artifacts exist.\")" + ] + }, + { + "cell_type": "markdown", + "id": "5f367a8d", + "metadata": {}, + "source": [ + "### Step 7 - Quick visual QA\n", + "\n", + "Use this for fast sanity checks only; final judgment comes from Notebook 02 simulator metrics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f43530b", + "metadata": {}, + "outputs": [], + "source": [ + "# Quick visual side-by-side snapshot\n", + "fig, axes = plt.subplots(2, 6, figsize=(14, 5))\n", + "for i in range(6):\n", + " axes[0, i].imshow(gen_designs[i], cmap=\"gray\", vmin=0, vmax=1)\n", + " axes[0, i].set_title(f\"gen {i}\")\n", + " axes[0, i].axis(\"off\")\n", + "\n", + " axes[1, i].imshow(baseline_designs[i], cmap=\"gray\", vmin=0, vmax=1)\n", + " axes[1, i].set_title(f\"base {i}\")\n", + " axes[1, i].axis(\"off\")\n", + "\n", + "fig.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "af80a49e", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "a6c079de", + "metadata": {}, + "source": [ + "## Next\n", + "\n", + "Proceed to Notebook 02 for physics-based evaluation and benchmark interpretation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f1794303", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/participant/02_evaluate_metrics.ipynb b/workshops/dcc26/participant/02_evaluate_metrics.ipynb new file mode 100644 index 0000000..f17fb16 --- /dev/null +++ b/workshops/dcc26/participant/02_evaluate_metrics.ipynb @@ -0,0 +1,627 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f51ce517", + "metadata": {}, + "source": [ + "# Notebook 02 (Participant): Evaluation + Metrics\n", + "\n", + "You will implement the evaluation layer and produce the evidence used for scientific comparison.\n" + ] + }, + { + "cell_type": "markdown", + "id": "71df2f22", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "9efcdfa1", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n", + "\n", + "### Public exercise legend\n", + "- `PUBLIC FILL-IN CELL`: complete this block directly.\n", + "- `CHECKPOINT`: verify before moving forward.\n", + "- Metric interpretation is as important as metric computation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "5fdfcb0f", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "This chapter answers: *did the generated designs actually improve engineering outcomes under benchmark simulation?*\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e82e4315", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"sqlitedict\", \"matplotlib\", \"tqdm\", \"tyro\", \"wandb\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "1a859723", + "metadata": {}, + "source": [ + "## Artifact loading\n", + "\n", + "Load artifacts from Notebook 01, optional W&B, or local auto-build fallback.\n", + "Do not proceed until artifact shapes/configs are confirmed.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f39cd896", + "metadata": {}, + "source": [ + "### Why these metrics matter for benchmarking\n", + "\n", + "Objective alone can overstate progress.\n", + "We include feasibility, diversity, and novelty proxies to reduce that blind spot.\n" + ] + }, + { + "cell_type": "markdown", + "id": "81355ad2", + "metadata": {}, + "source": [ + "### Step 1 - Resolve artifact source and recovery path\n", + "\n", + "Checkpoint: you can load `generated`, `baseline`, and `conditions` with matching sample counts.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1cc3e0e", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import random\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "try:\n", + " from engiopt.cgan_2d.cgan_2d import Generator as EngiOptCGAN2DGenerator\n", + "except ModuleNotFoundError as exc:\n", + " raise ModuleNotFoundError(\n", + " \"Could not import engiopt model class. Run the bootstrap cell first; on Colab, restart runtime after install if needed.\"\n", + " ) from exc\n", + "\n", + "USE_WANDB_ARTIFACTS = False\n", + "WANDB_PROJECT = \"dcc26-workshop\"\n", + "WANDB_ENTITY = None\n", + "WANDB_ARTIFACT_NAME = \"dcc26_beams2d_generated_artifacts\"\n", + "WANDB_ARTIFACT_ALIAS = \"latest\"\n", + "\n", + "# Self-heal path for workshop robustness\n", + "AUTO_BUILD_ARTIFACTS_IF_MISSING = True\n", + "\n", + "\n", + "def resolve_artifact_dir(create: bool = False) -> Path:\n", + " in_colab = \"google.colab\" in sys.modules\n", + " path = Path(\"/content/dcc26_artifacts\") if in_colab else Path(\"workshops/dcc26/artifacts\")\n", + " if create:\n", + " path.mkdir(parents=True, exist_ok=True)\n", + " return path\n", + "\n", + "\n", + "def build_artifacts_locally(\n", + " artifact_dir: Path,\n", + " seed: int = 7,\n", + " n_train: int = 512,\n", + " n_samples: int = 24,\n", + " epochs: int = 8,\n", + " batch_size: int = 64,\n", + " latent_dim: int = 32,\n", + ") -> None:\n", + " print(\"Building Notebook 01-style artifacts locally with EngiOpt...\")\n", + "\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " th.manual_seed(seed)\n", + " if th.cuda.is_available():\n", + " th.cuda.manual_seed_all(seed)\n", + "\n", + " device = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + " problem = Beams2D(seed=seed)\n", + " train_ds = problem.dataset[\"train\"]\n", + " test_ds = problem.dataset[\"test\"]\n", + " condition_keys = problem.conditions_keys\n", + "\n", + " rng = np.random.default_rng(seed)\n", + " subset_size = min(n_train, len(train_ds))\n", + " subset_idx = rng.choice(len(train_ds), size=subset_size, replace=False)\n", + "\n", + " conds_np = np.stack([np.array(train_ds[k])[subset_idx].astype(np.float32) for k in condition_keys], axis=1)\n", + " designs_np = np.array(train_ds[\"optimal_design\"])[subset_idx].astype(np.float32)\n", + " targets_np = designs_np * 2.0 - 1.0\n", + "\n", + " model = EngiOptCGAN2DGenerator(\n", + " latent_dim=latent_dim,\n", + " n_conds=conds_np.shape[1],\n", + " design_shape=problem.design_space.shape,\n", + " ).to(device)\n", + " optimizer = th.optim.Adam(model.parameters(), lr=1e-3)\n", + " criterion = nn.MSELoss()\n", + "\n", + " def sample_noise(batch: int) -> th.Tensor:\n", + " return th.randn((batch, latent_dim), device=device, dtype=th.float32)\n", + "\n", + " ds = TensorDataset(th.tensor(conds_np), th.tensor(targets_np))\n", + " dl = DataLoader(ds, batch_size=batch_size, shuffle=True)\n", + "\n", + " train_losses = []\n", + " for epoch in range(epochs):\n", + " model.train()\n", + " epoch_loss = 0.0\n", + " for cond_batch, target_batch in dl:\n", + " cond_batch = cond_batch.to(device)\n", + " target_batch = target_batch.to(device)\n", + "\n", + " pred = model(sample_noise(cond_batch.shape[0]), cond_batch)\n", + " loss = criterion(pred, target_batch)\n", + "\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " epoch_loss += float(loss.item())\n", + "\n", + " epoch_avg = epoch_loss / len(dl)\n", + " train_losses.append(epoch_avg)\n", + " print(f\"bootstrap epoch {epoch + 1:02d}/{epochs} - loss: {epoch_avg:.4f}\")\n", + "\n", + " sample_count = min(n_samples, len(test_ds))\n", + " selected = rng.choice(len(test_ds), size=sample_count, replace=False)\n", + " test_conds = np.stack([np.array(test_ds[k])[selected].astype(np.float32) for k in condition_keys], axis=1)\n", + " baseline_designs = np.array(test_ds[\"optimal_design\"])[selected].astype(np.float32)\n", + "\n", + " model.eval()\n", + " with th.no_grad():\n", + " tanh_out = model(sample_noise(sample_count), th.tensor(test_conds, device=device))\n", + " gen_designs_t = ((tanh_out.clamp(-1.0, 1.0) + 1.0) / 2.0).clamp(0.0, 1.0)\n", + " gen_designs = gen_designs_t.detach().cpu().numpy().astype(np.float32)\n", + "\n", + " conditions_records = []\n", + " for i in range(sample_count):\n", + " rec = {}\n", + " for j, k in enumerate(condition_keys):\n", + " v = test_conds[i, j]\n", + " rec[k] = bool(v) if k == \"overhang_constraint\" else float(v)\n", + " conditions_records.append(rec)\n", + "\n", + " artifact_dir.mkdir(parents=True, exist_ok=True)\n", + " np.save(artifact_dir / \"generated_designs.npy\", gen_designs)\n", + " np.save(artifact_dir / \"baseline_designs.npy\", baseline_designs)\n", + " with open(artifact_dir / \"conditions.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(conditions_records, f, indent=2)\n", + "\n", + " pd.DataFrame({\"epoch\": np.arange(1, len(train_losses) + 1), \"train_loss\": train_losses}).to_csv(\n", + " artifact_dir / \"training_history.csv\", index=False\n", + " )\n", + "\n", + " th.save(\n", + " {\n", + " \"model\": model.state_dict(),\n", + " \"condition_keys\": condition_keys,\n", + " \"latent_dim\": latent_dim,\n", + " \"model_family\": \"engiopt.cgan_2d.Generator\",\n", + " },\n", + " artifact_dir / \"engiopt_cgan2d_generator_supervised.pt\",\n", + " )\n", + "\n", + " print(\"Built artifacts at\", artifact_dir)\n", + "\n", + "\n", + "ARTIFACT_DIR = resolve_artifact_dir(create=True)\n", + "required = [\n", + " ARTIFACT_DIR / \"generated_designs.npy\",\n", + " ARTIFACT_DIR / \"baseline_designs.npy\",\n", + " ARTIFACT_DIR / \"conditions.json\",\n", + "]\n", + "\n", + "if not all(p.exists() for p in required):\n", + " if USE_WANDB_ARTIFACTS:\n", + " try:\n", + " import wandb\n", + "\n", + " run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, job_type=\"artifact-download\", reinit=True)\n", + " if WANDB_ENTITY:\n", + " artifact_ref = f\"{WANDB_ENTITY}/{WANDB_PROJECT}/{WANDB_ARTIFACT_NAME}:{WANDB_ARTIFACT_ALIAS}\"\n", + " else:\n", + " artifact_ref = f\"{WANDB_PROJECT}/{WANDB_ARTIFACT_NAME}:{WANDB_ARTIFACT_ALIAS}\"\n", + " artifact = run.use_artifact(artifact_ref, type=\"dataset\")\n", + " artifact.download(root=str(ARTIFACT_DIR))\n", + " run.finish()\n", + " print(\"Downloaded artifacts from W&B to\", ARTIFACT_DIR)\n", + " except Exception as exc:\n", + " if AUTO_BUILD_ARTIFACTS_IF_MISSING:\n", + " print(\"W&B download failed; switching to local artifact build:\", exc)\n", + " build_artifacts_locally(ARTIFACT_DIR)\n", + " else:\n", + " raise FileNotFoundError(\n", + " \"Artifacts missing locally and W&B download failed. \"\n", + " \"Run Notebook 01 first or disable USE_WANDB_ARTIFACTS. \"\n", + " f\"Details: {exc}\"\n", + " ) from exc\n", + " elif AUTO_BUILD_ARTIFACTS_IF_MISSING:\n", + " build_artifacts_locally(ARTIFACT_DIR)\n", + " else:\n", + " missing = \"\\n\".join(f\"- {p}\" for p in required if not p.exists())\n", + " raise FileNotFoundError(\n", + " \"Notebook 01 artifacts not found. Run Notebook 01 first (including export cell), \"\n", + " \"or enable USE_WANDB_ARTIFACTS to fetch from W&B. Missing files:\\\\n\" + missing\n", + " )\n", + "\n", + "print(\"using artifact dir:\", ARTIFACT_DIR)\n", + "\n", + "gen_designs = np.load(ARTIFACT_DIR / \"generated_designs.npy\")\n", + "baseline_designs = np.load(ARTIFACT_DIR / \"baseline_designs.npy\")\n", + "with open(ARTIFACT_DIR / \"conditions.json\", encoding=\"utf-8\") as f:\n", + " conditions = json.load(f)\n", + "\n", + "print(\"generated:\", gen_designs.shape)\n", + "print(\"baseline:\", baseline_designs.shape)\n", + "print(\"conditions:\", len(conditions))" + ] + }, + { + "cell_type": "markdown", + "id": "82f21c6f", + "metadata": {}, + "source": [ + "### Step 2 - Implement per-sample evaluation (PUBLIC FILL-IN)\n", + "\n", + "Compute constraint violations + objective values for generated and baseline designs under identical conditions.\n", + "\n", + "Success criteria:\n", + "- one row per sample,\n", + "- objective values and violation counts both captured.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "553ff3c9", + "metadata": {}, + "outputs": [], + "source": [ + "problem = Beams2D(seed=7)\n", + "\n", + "# PUBLIC FILL-IN CELL 02-A\n", + "# Goal: evaluate each sample pair with identical condition config.\n", + "\n", + "rows = []\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "# for i, (g, b, cfg_raw) in enumerate(zip(gen_designs, baseline_designs, conditions, strict=True)):\n", + "# cfg = dict(cfg_raw)\n", + "# g_viol = problem.check_constraints(design=g, config=cfg)\n", + "# b_viol = problem.check_constraints(design=b, config=cfg)\n", + "#\n", + "# # reset between simulations for reproducibility / simulator hygiene\n", + "# problem.reset(seed=7 + i)\n", + "# g_obj = float(problem.simulate(design=g, config=cfg))\n", + "# problem.reset(seed=7 + i)\n", + "# b_obj = float(problem.simulate(design=b, config=cfg))\n", + "#\n", + "# rows.append({\n", + "# 'sample': i,\n", + "# 'gen_obj': g_obj,\n", + "# 'base_obj': b_obj,\n", + "# 'gen_minus_base': g_obj - b_obj,\n", + "# 'gen_violations': len(g_viol),\n", + "# 'base_violations': len(b_viol),\n", + "# })\n", + "raise NotImplementedError(\"Implement per-sample evaluation loop\")\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "results = pd.DataFrame(rows)\n", + "results.head()\n", + "\n", + "# CHECKPOINT\n", + "expected_cols = {\"sample\", \"gen_obj\", \"base_obj\", \"gen_minus_base\", \"gen_violations\", \"base_violations\"}\n", + "missing_cols = expected_cols.difference(results.columns)\n", + "assert not missing_cols, f\"Missing result columns: {missing_cols}\"\n", + "assert len(results) == len(gen_designs), \"results must have one row per sample\"\n", + "print(\"Checkpoint passed: per-sample evaluation table is complete.\")" + ] + }, + { + "cell_type": "markdown", + "id": "fbcc2466", + "metadata": {}, + "source": [ + "### Step 3 - Implement summary metrics (PUBLIC FILL-IN)\n", + "\n", + "Report objective, feasibility, diversity, and novelty in a single table.\n", + "\n", + "Success criteria:\n", + "- `summary_df` has one row,\n", + "- each metric has a clear interpretation for workshop discussion.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0115d7", + "metadata": {}, + "outputs": [], + "source": [ + "# PUBLIC FILL-IN CELL 02-B\n", + "# Goal: aggregate per-sample metrics into one benchmark summary row.\n", + "\n", + "\n", + "def mean_pairwise_l2(designs: np.ndarray) -> float:\n", + " flat = designs.reshape(designs.shape[0], -1)\n", + " n = flat.shape[0]\n", + " if n < 2:\n", + " return 0.0\n", + " dists = []\n", + " for i in range(n):\n", + " for j in range(i + 1, n):\n", + " dists.append(float(np.linalg.norm(flat[i] - flat[j])))\n", + " return float(np.mean(dists))\n", + "\n", + "\n", + "def mean_nn_distance_to_reference(designs: np.ndarray, reference_designs: np.ndarray) -> float:\n", + " q = designs.reshape(designs.shape[0], -1)\n", + " r = reference_designs.reshape(reference_designs.shape[0], -1)\n", + " nn_dists = []\n", + " for i in range(q.shape[0]):\n", + " d = np.linalg.norm(r - q[i][None, :], axis=1)\n", + " nn_dists.append(float(np.min(d)))\n", + " return float(np.mean(nn_dists))\n", + "\n", + "\n", + "# START FILL ---------------------------------------------------------------\n", + "# Build one summary dict with at least these keys:\n", + "# - n_samples\n", + "# - gen_obj_mean\n", + "# - base_obj_mean\n", + "# - objective_gap_mean\n", + "# - improvement_rate\n", + "# - gen_violation_ratio\n", + "# - base_violation_ratio\n", + "# - gen_feasible_rate\n", + "# - gen_diversity_l2\n", + "# - gen_novelty_to_train_l2\n", + "raise NotImplementedError(\"Implement summary metric dictionary + summary_df\")\n", + "# END FILL -----------------------------------------------------------------\n", + "\n", + "# CHECKPOINT\n", + "assert \"summary_df\" in locals(), \"Define summary_df\"\n", + "assert len(summary_df) == 1, \"summary_df should be one-row summary\"\n", + "required_summary = {\n", + " \"n_samples\",\n", + " \"gen_obj_mean\",\n", + " \"base_obj_mean\",\n", + " \"objective_gap_mean\",\n", + " \"improvement_rate\",\n", + " \"gen_violation_ratio\",\n", + " \"base_violation_ratio\",\n", + " \"gen_feasible_rate\",\n", + " \"gen_diversity_l2\",\n", + " \"gen_novelty_to_train_l2\",\n", + "}\n", + "missing = required_summary.difference(summary_df.columns)\n", + "assert not missing, f\"Missing summary columns: {missing}\"\n", + "print(\"Checkpoint passed: summary table is ready for export/discussion.\")" + ] + }, + { + "cell_type": "markdown", + "id": "e7d2eb8c", + "metadata": {}, + "source": [ + "### Step 4 - Export evidence artifacts\n", + "\n", + "Persist outputs as audit-ready artifacts, not ephemeral notebook prints.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f441a31b", + "metadata": {}, + "outputs": [], + "source": [ + "# Export metrics and figures\n", + "results_path = ARTIFACT_DIR / \"per_sample_metrics.csv\"\n", + "summary_path = ARTIFACT_DIR / \"metrics_summary.csv\"\n", + "hist_path = ARTIFACT_DIR / \"objective_histogram.png\"\n", + "grid_path = ARTIFACT_DIR / \"design_grid.png\"\n", + "scatter_path = ARTIFACT_DIR / \"objective_scatter.png\"\n", + "\n", + "results.to_csv(results_path, index=False)\n", + "summary_df.to_csv(summary_path, index=False)\n", + "\n", + "fig, ax = plt.subplots(figsize=(7, 4))\n", + "ax.hist(results[\"gen_obj\"], bins=10, alpha=0.7, label=\"generated\")\n", + "ax.hist(results[\"base_obj\"], bins=10, alpha=0.7, label=\"baseline\")\n", + "ax.set_xlabel(\"Compliance objective (lower is better)\")\n", + "ax.set_ylabel(\"Count\")\n", + "ax.set_title(\"Generated vs baseline objective distribution\")\n", + "ax.legend()\n", + "fig.tight_layout()\n", + "fig.savefig(hist_path, dpi=150)\n", + "plt.show()\n", + "\n", + "fig2, ax2 = plt.subplots(figsize=(5, 5))\n", + "ax2.scatter(results[\"base_obj\"], results[\"gen_obj\"], alpha=0.8)\n", + "min_v = min(results[\"base_obj\"].min(), results[\"gen_obj\"].min())\n", + "max_v = max(results[\"base_obj\"].max(), results[\"gen_obj\"].max())\n", + "ax2.plot([min_v, max_v], [min_v, max_v], \"--\", color=\"black\", linewidth=1)\n", + "ax2.set_xlabel(\"Baseline objective\")\n", + "ax2.set_ylabel(\"Generated objective\")\n", + "ax2.set_title(\"Per-sample objective comparison\")\n", + "fig2.tight_layout()\n", + "fig2.savefig(scatter_path, dpi=150)\n", + "plt.show()\n", + "\n", + "print(\"Saved:\")\n", + "print(\"-\", results_path)\n", + "print(\"-\", summary_path)\n", + "print(\"-\", hist_path)\n", + "print(\"-\", scatter_path)\n", + "\n", + "# Optional advanced extension:\n", + "# if USE_WANDB_ARTIFACTS:\n", + "# log summary metrics, tables, and images to W&B." + ] + }, + { + "cell_type": "markdown", + "id": "a70e02fe", + "metadata": {}, + "source": [ + "### Step 5 - Visual comparison for interpretation\n", + "\n", + "Use visuals to interpret outliers and reconcile metric-level contradictions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ae72a84", + "metadata": {}, + "outputs": [], + "source": [ + "# Visual side-by-side sample grid\n", + "fig, axes = plt.subplots(3, 4, figsize=(12, 8))\n", + "for i, ax in enumerate(axes.ravel()):\n", + " if i >= 12:\n", + " break\n", + " pair_idx = i // 2\n", + " if i % 2 == 0:\n", + " ax.imshow(gen_designs[pair_idx], cmap=\"gray\", vmin=0, vmax=1)\n", + " ax.set_title(f\"gen {pair_idx}\")\n", + " else:\n", + " ax.imshow(baseline_designs[pair_idx], cmap=\"gray\", vmin=0, vmax=1)\n", + " ax.set_title(f\"base {pair_idx}\")\n", + " ax.axis(\"off\")\n", + "fig.tight_layout()\n", + "fig.savefig(grid_path, dpi=150)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e1864c41", + "metadata": {}, + "source": [ + "## Interpretation hints\n", + "\n", + "Strong results usually balance objective quality **and** feasibility.\n", + "Use diversity/novelty to discuss exploration vs imitation behavior.\n" + ] + }, + { + "cell_type": "markdown", + "id": "9fc8ab15", + "metadata": {}, + "source": [ + "## Discussion bridge to workshop breakout\n", + "\n", + "Bring one concrete claim and one uncertainty to discussion.\n", + "Example: “Objective improved, but feasibility degraded under stricter conditions.”\n" + ] + }, + { + "cell_type": "markdown", + "id": "e0bf0166", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "0905c130", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/participant/03_add_new_problem_scaffold.ipynb b/workshops/dcc26/participant/03_add_new_problem_scaffold.ipynb new file mode 100644 index 0000000..f9af837 --- /dev/null +++ b/workshops/dcc26/participant/03_add_new_problem_scaffold.ipynb @@ -0,0 +1,816 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2c1598d9", + "metadata": {}, + "source": [ + "# Notebook 03 (Participant): Add a New Problem Scaffold\n", + "\n", + "You will implement a robotics co-design problem contract and evaluate whether it is benchmark-ready.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f05bea02", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "a3ff5574", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n", + "\n", + "### Public exercise legend\n", + "- `PUBLIC FILL-IN`: implement this method block.\n", + "- Follow the fill order: constraints -> simulator build -> rollout -> wrappers.\n", + "- Run smoke test only after all TODO blocks are implemented.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f451e420", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "This chapter is about benchmark design quality, not model training speed.\n" + ] + }, + { + "cell_type": "markdown", + "id": "83b36f03", + "metadata": {}, + "source": [ + "## What makes a new problem benchmark-ready\n", + "\n", + "A publishable benchmark needs explicit representation, constraints, objectives, simulator semantics, and reproducibility metadata.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06fe16ca", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"matplotlib\", \"gymnasium\", \"pybullet\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "064e668b", + "metadata": {}, + "source": [ + "### Step 1 - Import scaffold dependencies\n", + "\n", + "Ensure all required interfaces are visible before class implementation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39df1bf6", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "from dataclasses import dataclass\n", + "from typing import Annotated\n", + "\n", + "import numpy as np\n", + "from gymnasium import spaces\n", + "\n", + "from engibench.constraint import bounded\n", + "from engibench.constraint import constraint\n", + "from engibench.core import ObjectiveDirection\n", + "from engibench.core import OptiStep\n", + "from engibench.core import Problem\n", + "\n", + "import pybullet as p" + ] + }, + { + "cell_type": "markdown", + "id": "d4a22b27", + "metadata": {}, + "source": [ + "### Step 2 - Implement PyBullet manipulator co-design problem contract (PUBLIC FILL-IN)\n", + "\n", + "Complete each required method with deterministic behavior and clear failure messages.\n", + "\n", + "Recommended fill order:\n", + "1. Constraints in `__init__`\n", + "2. `_build_robot`\n", + "3. IK/FK helpers\n", + "4. `_rollout`\n", + "5. `simulate`, `optimize`, `render`, `random_design`\n", + "\n", + "Success criteria:\n", + "- smoke test prints objectives and optimization progress,\n", + "- render figure is interpretable (path/error/torque views).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3562f034", + "metadata": {}, + "outputs": [], + "source": [ + "class PlanarManipulatorCoDesignProblem(Problem[np.ndarray]):\n", + " \"\"\"Robotics co-design scaffold using a real PyBullet rollout loop.\"\"\"\n", + "\n", + " version = 0\n", + " objectives = (\n", + " (\"final_tracking_error_m\", ObjectiveDirection.MINIMIZE),\n", + " (\"actuation_energy_j\", ObjectiveDirection.MINIMIZE),\n", + " )\n", + "\n", + " @dataclass\n", + " class Conditions:\n", + " target_x: Annotated[float, bounded(lower=0.20, upper=1.35)] = 0.85\n", + " target_y: Annotated[float, bounded(lower=0.05, upper=1.20)] = 0.45\n", + " payload_kg: Annotated[float, bounded(lower=0.0, upper=2.0)] = 0.8\n", + " disturbance_scale: Annotated[float, bounded(lower=0.0, upper=0.30)] = 0.05\n", + "\n", + " @dataclass\n", + " class Config(Conditions):\n", + " sim_steps: Annotated[int, bounded(lower=60, upper=1200)] = 240\n", + " dt: Annotated[float, bounded(lower=1e-4, upper=0.05)] = 1.0 / 120.0\n", + " torque_limit: Annotated[float, bounded(lower=1.0, upper=50.0)] = 12.0\n", + " max_iter: Annotated[int, bounded(lower=1, upper=300)] = 60\n", + "\n", + " dataset_id = \"IDEALLab/planar_manipulator_codesign_v0\" # placeholder for future dataset integration\n", + " container_id = None\n", + "\n", + " def __init__(self, seed: int = 0, **kwargs):\n", + " super().__init__(seed=seed)\n", + " self.config = self.Config(**kwargs)\n", + " self.conditions = self.Conditions(\n", + " target_x=self.config.target_x,\n", + " target_y=self.config.target_y,\n", + " payload_kg=self.config.payload_kg,\n", + " disturbance_scale=self.config.disturbance_scale,\n", + " )\n", + "\n", + " # Design vector = [link1_m, link2_m, motor_strength, kp, kd, damping]\n", + " self.design_space = spaces.Box(\n", + " low=np.array([0.25, 0.20, 2.0, 5.0, 0.2, 0.0], dtype=np.float32),\n", + " high=np.array([1.00, 0.95, 30.0, 120.0, 18.0, 1.5], dtype=np.float32),\n", + " dtype=np.float32,\n", + " )\n", + "\n", + " # PUBLIC FILL-IN 03-A1: constraints\n", + " # START FILL -------------------------------------------------------\n", + " # Define two @constraint functions:\n", + " # (1) reachable_workspace(design, target_x, target_y, **_) -> assert reachable radius\n", + " # (2) gain_consistency(design, **_) -> assert kd is reasonable for kp\n", + " # Then assign: self.design_constraints = [reachable_workspace, gain_consistency]\n", + " raise NotImplementedError(\"Implement __init__ constraints\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def _build_robot(self, l1: float, l2: float, payload_kg: float, damping: float) -> tuple[int, int]:\n", + " # PUBLIC FILL-IN 03-A2: build robot\n", + " # START FILL -------------------------------------------------------\n", + " # Required behavior:\n", + " # - reset simulation and gravity\n", + " # - create a 2-link articulated body\n", + " # - apply damping to joints\n", + " # - return (robot_id, end_effector_link_index)\n", + " raise NotImplementedError(\"Implement _build_robot\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def _inverse_kinematics_2link(self, x: float, y: float, l1: float, l2: float) -> tuple[float, float]:\n", + " # PUBLIC FILL-IN 03-A3: 2-link IK\n", + " # START FILL -------------------------------------------------------\n", + " # Implement stable closed-form IK with clipping for numerical robustness.\n", + " raise NotImplementedError(\"Implement _inverse_kinematics_2link\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def _forward_kinematics_2link(self, q1: float, q2: float, l1: float, l2: float) -> tuple[float, float]:\n", + " # PUBLIC FILL-IN 03-A4: 2-link FK\n", + " # START FILL -------------------------------------------------------\n", + " # Return end-effector (x, y) from joint angles and link lengths.\n", + " raise NotImplementedError(\"Implement _forward_kinematics_2link\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def _rollout(self, design: np.ndarray, cfg: dict, return_trace: bool = False):\n", + " # PUBLIC FILL-IN 03-A5: rollout + objective computation\n", + " # START FILL -------------------------------------------------------\n", + " # Required outputs:\n", + " # - objective vector: [final_tracking_error_m, actuation_energy_j]\n", + " # - if return_trace=True: dict with ee_trace, err_trace, tau_trace, target\n", + " # Tips:\n", + " # - connect with p.DIRECT, always disconnect in finally\n", + " # - apply optional disturbances from cfg['disturbance_scale']\n", + " raise NotImplementedError(\"Implement _rollout\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def simulate(self, design: np.ndarray, config: dict | None = None) -> np.ndarray:\n", + " # PUBLIC FILL-IN 03-A6: benchmark simulation wrapper\n", + " # START FILL -------------------------------------------------------\n", + " # Merge cfg, clip design to bounds, return objective vector from _rollout.\n", + " raise NotImplementedError(\"Implement simulate\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def optimize(self, starting_point: np.ndarray, config: dict | None = None):\n", + " # PUBLIC FILL-IN 03-A7: simple deterministic optimizer\n", + " # START FILL -------------------------------------------------------\n", + " # Implement local search and return (best_design, history[OptiStep]).\n", + " # Keep deterministic behavior via self.np_random.\n", + " raise NotImplementedError(\"Implement optimize\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def render(self, design: np.ndarray, *, open_window: bool = False):\n", + " # PUBLIC FILL-IN 03-A8: interpretation plotting\n", + " # START FILL -------------------------------------------------------\n", + " # Create 4 panels:\n", + " # (1) design vars, (2) task-space path + target,\n", + " # (3) error over time, (4) torque over time.\n", + " raise NotImplementedError(\"Implement render\")\n", + " # END FILL ---------------------------------------------------------\n", + "\n", + " def random_design(self):\n", + " # PUBLIC FILL-IN 03-A9: random design sampler\n", + " # START FILL -------------------------------------------------------\n", + " # Return a random design in bounds and dummy reward -1.\n", + " raise NotImplementedError(\"Implement random_design\")\n", + " # END FILL ---------------------------------------------------------" + ] + }, + { + "cell_type": "markdown", + "id": "363a7164", + "metadata": {}, + "source": [ + "### Step 3 - Smoke-test your scaffold\n", + "\n", + "Run this only after completing all PUBLIC FILL-IN blocks above.\n", + "\n", + "What success looks like:\n", + "- non-empty optimization history,\n", + "- final objective improves over initial objective,\n", + "- 4-panel figure renders without error.\n", + "\n", + "If the cell fails:\n", + "- re-check one method at a time in fill-order,\n", + "- especially `_rollout` and `simulate` shape/return semantics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec683bc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Smoke test (run after implementing all PUBLIC FILL-IN blocks)\n", + "problem = PlanarManipulatorCoDesignProblem(\n", + " seed=42,\n", + " target_x=0.9,\n", + " target_y=0.45,\n", + " payload_kg=0.8,\n", + " disturbance_scale=0.04,\n", + " sim_steps=220,\n", + " max_iter=40,\n", + ")\n", + "start, _ = problem.random_design()\n", + "\n", + "cfg = {\n", + " \"target_x\": 0.9,\n", + " \"target_y\": 0.45,\n", + " \"payload_kg\": 0.8,\n", + " \"disturbance_scale\": 0.04,\n", + " \"sim_steps\": 220,\n", + " \"dt\": 1.0 / 120.0,\n", + " \"torque_limit\": 12.0,\n", + " \"max_iter\": 40,\n", + "}\n", + "\n", + "print(\"design space:\", problem.design_space)\n", + "print(\"objectives:\", problem.objectives)\n", + "print(\"conditions:\", problem.conditions)\n", + "\n", + "viol = problem.check_constraints(start, config=cfg)\n", + "print(\"constraint violations:\", len(viol))\n", + "\n", + "obj0 = problem.simulate(start, config=cfg)\n", + "opt_design, history = problem.optimize(start, config=cfg)\n", + "objf = problem.simulate(opt_design, config=cfg)\n", + "\n", + "print(\"initial objectives [tracking_error_m, energy_J]:\", obj0.tolist())\n", + "print(\"final objectives [tracking_error_m, energy_J]:\", objf.tolist())\n", + "print(\"optimization steps:\", len(history))\n", + "print(\"How to read plots: vars | task-space path | error timeline | torque timeline\")\n", + "\n", + "# CHECKPOINT\n", + "assert len(history) > 0, \"Optimization history should not be empty\"\n", + "assert np.all(np.isfinite(obj0)), \"Initial objective contains non-finite values\"\n", + "assert np.all(np.isfinite(objf)), \"Final objective contains non-finite values\"\n", + "\n", + "problem.render(opt_design)" + ] + }, + { + "cell_type": "markdown", + "id": "5ab2097c", + "metadata": {}, + "source": [ + "## Mapping to real EngiBench contributions\n", + "\n", + "Translate this robotics co-design scaffold into domain-specific simulators and datasets (robotics, controls, etc.) with documented assumptions.\n" + ] + }, + { + "cell_type": "markdown", + "id": "e2e5bd63", + "metadata": {}, + "source": [ + "## Contribution checklist\n", + "\n", + "Before proposing a new problem, verify data provenance, split policy, evaluation protocol, and reporting templates.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ea79fd4f", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "7a014b05", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + }, + { + "cell_type": "markdown", + "id": "ee70c214", + "metadata": {}, + "source": [ + "## Optional extension - Build dataset and train an EngiOpt generative model\n", + "\n", + "This extension runs a full offline loop using an **existing EngiOpt model** (`cgan_1d`) on top of your custom simulator problem:\n", + "\n", + "1. Generate a feasible dataset from simulator rollouts.\n", + "2. Keep a top-performing subset.\n", + "3. Train EngiOpt `cgan_1d` (`Generator` + `Discriminator`) for conditional generation.\n", + "4. Compare generated designs vs a random-design baseline.\n", + "\n", + "Why this is useful:\n", + "- It demonstrates reuse of existing model infrastructure from EngiOpt.\n", + "- It clarifies how to adapt EngiOpt model classes to new, custom problem scaffolds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af8e43e7", + "metadata": {}, + "outputs": [], + "source": [ + "# Optional extension controls (safe defaults)\n", + "from pathlib import Path\n", + "import sys\n", + "\n", + "import numpy as np\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "RUN_OPTIONAL_SECTION = False # Set True to run this optional extension\n", + "N_FEASIBLE_SAMPLES = 260\n", + "TOP_FRACTION = 0.35\n", + "EPOCHS = 30\n", + "BATCH_SIZE = 64\n", + "LATENT_DIM = 8\n", + "FAST_SIM_CFG = {\"sim_steps\": 80, \"dt\": 1.0 / 120.0}\n", + "EVAL_SAMPLES = 40\n", + "\n", + "if \"problem\" not in globals():\n", + " problem = PlanarManipulatorCoDesignProblem(seed=7)\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + " OPTIONAL_ARTIFACT_DIR = Path(\"/content/dcc26_optional_artifacts\")\n", + "else:\n", + " OPTIONAL_ARTIFACT_DIR = Path(\"workshops/dcc26/optional_artifacts\")\n", + "OPTIONAL_ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(f\"Optional artifacts dir: {OPTIONAL_ARTIFACT_DIR.resolve()}\")\n", + "print(\"Optional section enabled:\" if RUN_OPTIONAL_SECTION else \"Optional section disabled:\", RUN_OPTIONAL_SECTION)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a16f8e2", + "metadata": {}, + "outputs": [], + "source": [ + "# Build offline dataset from simulator rollouts\n", + "rng = np.random.default_rng(123)\n", + "\n", + "\n", + "def sample_condition_dict() -> dict:\n", + " return {\n", + " \"target_x\": float(rng.uniform(0.20, 1.35)),\n", + " \"target_y\": float(rng.uniform(0.05, 1.20)),\n", + " \"payload_kg\": float(rng.uniform(0.0, 2.0)),\n", + " \"disturbance_scale\": float(rng.uniform(0.0, 0.30)),\n", + " }\n", + "\n", + "\n", + "def cond_to_vec(cfg: dict) -> np.ndarray:\n", + " return np.array(\n", + " [\n", + " cfg[\"target_x\"],\n", + " cfg[\"target_y\"],\n", + " cfg[\"payload_kg\"],\n", + " cfg[\"disturbance_scale\"],\n", + " ],\n", + " dtype=np.float32,\n", + " )\n", + "\n", + "\n", + "def objective_score(obj: np.ndarray) -> float:\n", + " return float(obj[0] + 0.02 * obj[1])\n", + "\n", + "\n", + "def make_dataset(problem_obj, n_feasible: int):\n", + " designs, conds, objs = [], [], []\n", + " max_attempts = n_feasible * 8\n", + " attempts = 0\n", + "\n", + " while len(designs) < n_feasible and attempts < max_attempts:\n", + " attempts += 1\n", + " d, _ = problem_obj.random_design()\n", + " cfg = sample_condition_dict()\n", + "\n", + " if len(problem_obj.check_constraints(d, cfg)) > 0:\n", + " continue\n", + "\n", + " obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " designs.append(d.astype(np.float32))\n", + " conds.append(cond_to_vec(cfg))\n", + " objs.append(obj.astype(np.float32))\n", + "\n", + " if len(designs) % 40 == 0:\n", + " print(f\"Collected feasible samples: {len(designs)}/{n_feasible}\")\n", + "\n", + " if len(designs) < max(48, n_feasible // 3):\n", + " raise RuntimeError(f\"Not enough feasible samples ({len(designs)}).\")\n", + "\n", + " designs = np.stack(designs)\n", + " conds = np.stack(conds)\n", + " objs = np.stack(objs)\n", + " scores = np.array([objective_score(o) for o in objs], dtype=np.float32)\n", + "\n", + " keep_n = max(48, int(TOP_FRACTION * len(scores)))\n", + " top_idx = np.argsort(scores)[:keep_n]\n", + "\n", + " data = {\n", + " \"designs_all\": designs,\n", + " \"conditions_all\": conds,\n", + " \"objectives_all\": objs,\n", + " \"scores_all\": scores,\n", + " \"designs_top\": designs[top_idx],\n", + " \"conditions_top\": conds[top_idx],\n", + " \"objectives_top\": objs[top_idx],\n", + " \"scores_top\": scores[top_idx],\n", + " }\n", + " return data\n", + "\n", + "\n", + "if RUN_OPTIONAL_SECTION:\n", + " dataset = make_dataset(problem, N_FEASIBLE_SAMPLES)\n", + " np.savez(OPTIONAL_ARTIFACT_DIR / \"manipulator_dataset.npz\", **dataset)\n", + " print(\"Saved dataset:\", OPTIONAL_ARTIFACT_DIR / \"manipulator_dataset.npz\")\n", + " print(\"All samples:\", dataset[\"designs_all\"].shape[0], \"| Top samples:\", dataset[\"designs_top\"].shape[0])\n", + "else:\n", + " dataset = None\n", + " print(\"Skipped dataset creation. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "markdown", + "id": "0231e825", + "metadata": {}, + "source": [ + "### Optional model - EngiOpt `cgan_1d`\n", + "\n", + "This cell reuses `engiopt.cgan_1d.cgan_1d` classes directly:\n", + "- `Normalizer`\n", + "- `Generator`\n", + "- `Discriminator`\n", + "\n", + "Adapter note:\n", + "- `Discriminator` in this module expects module-level `design_shape` and `n_conds` symbols.\n", + "- We set those explicitly before instantiation to keep behavior aligned with the original script.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14ce7c39", + "metadata": {}, + "outputs": [], + "source": [ + "# Train EngiOpt cgan_1d on the top-performing subset\n", + "if RUN_OPTIONAL_SECTION:\n", + " import engiopt.cgan_1d.cgan_1d as cgan1d\n", + "\n", + " device = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + "\n", + " x_cond = dataset[\"conditions_top\"].astype(np.float32)\n", + " y_design = dataset[\"designs_top\"].astype(np.float32)\n", + "\n", + " cond_t = th.tensor(x_cond, dtype=th.float32, device=device)\n", + " design_t = th.tensor(y_design, dtype=th.float32, device=device)\n", + "\n", + " cond_min = cond_t.amin(dim=0)\n", + " cond_max = cond_t.amax(dim=0)\n", + " design_min = design_t.amin(dim=0)\n", + " design_max = design_t.amax(dim=0)\n", + "\n", + " conds_normalizer = cgan1d.Normalizer(cond_min, cond_max)\n", + " design_normalizer = cgan1d.Normalizer(design_min, design_max)\n", + "\n", + " design_shape = (design_t.shape[1],)\n", + " n_conds = cond_t.shape[1]\n", + "\n", + " # Compatibility shim for cgan_1d.Discriminator internal references.\n", + " cgan1d.design_shape = design_shape\n", + " cgan1d.n_conds = n_conds\n", + "\n", + " generator = cgan1d.Generator(\n", + " latent_dim=LATENT_DIM,\n", + " n_conds=n_conds,\n", + " design_shape=design_shape,\n", + " design_normalizer=design_normalizer,\n", + " conds_normalizer=conds_normalizer,\n", + " ).to(device)\n", + "\n", + " discriminator = cgan1d.Discriminator(\n", + " conds_normalizer=conds_normalizer,\n", + " design_normalizer=design_normalizer,\n", + " ).to(device)\n", + "\n", + " loader = DataLoader(TensorDataset(design_t, cond_t), batch_size=BATCH_SIZE, shuffle=True)\n", + "\n", + " adv_loss = nn.BCELoss()\n", + " opt_g = th.optim.Adam(generator.parameters(), lr=2e-4, betas=(0.5, 0.999))\n", + " opt_d = th.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.5, 0.999))\n", + "\n", + " g_hist, d_hist = [], []\n", + "\n", + " for epoch in range(1, EPOCHS + 1):\n", + " g_epoch, d_epoch, n_steps = 0.0, 0.0, 0\n", + " for real_design, cond in loader:\n", + " bs = real_design.shape[0]\n", + " valid = th.ones((bs, 1), device=device)\n", + " fake = th.zeros((bs, 1), device=device)\n", + "\n", + " # Generator update\n", + " opt_g.zero_grad()\n", + " z = th.randn((bs, LATENT_DIM), device=device)\n", + " gen_design = generator(z, cond)\n", + " g_loss = adv_loss(discriminator(gen_design, cond), valid)\n", + " g_loss.backward()\n", + " opt_g.step()\n", + "\n", + " # Discriminator update\n", + " opt_d.zero_grad()\n", + " real_loss = adv_loss(discriminator(real_design, cond), valid)\n", + " fake_loss = adv_loss(discriminator(gen_design.detach(), cond), fake)\n", + " d_loss = 0.5 * (real_loss + fake_loss)\n", + " d_loss.backward()\n", + " opt_d.step()\n", + "\n", + " g_epoch += float(g_loss.item())\n", + " d_epoch += float(d_loss.item())\n", + " n_steps += 1\n", + "\n", + " g_hist.append(g_epoch / max(1, n_steps))\n", + " d_hist.append(d_epoch / max(1, n_steps))\n", + " if epoch == 1 or epoch % 5 == 0 or epoch == EPOCHS:\n", + " print(f\"Epoch {epoch:02d}/{EPOCHS} | g_loss={g_hist[-1]:.6f} | d_loss={d_hist[-1]:.6f}\")\n", + "\n", + " th.save(\n", + " {\n", + " \"generator\": generator.state_dict(),\n", + " \"discriminator\": discriminator.state_dict(),\n", + " \"cond_min\": cond_min.cpu(),\n", + " \"cond_max\": cond_max.cpu(),\n", + " \"design_min\": design_min.cpu(),\n", + " \"design_max\": design_max.cpu(),\n", + " },\n", + " OPTIONAL_ARTIFACT_DIR / \"engiopt_cgan1d_weights.pt\",\n", + " )\n", + " print(\"Saved model:\", OPTIONAL_ARTIFACT_DIR / \"engiopt_cgan1d_weights.pt\")\n", + "else:\n", + " generator, discriminator = None, None\n", + " g_hist, d_hist = [], []\n", + " device = th.device(\"cpu\")\n", + " print(\"Skipped model training. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bea5477f", + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate generated designs vs random baseline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "def sample_baseline(problem_obj, cfg: dict, trials: int = 8):\n", + " best_obj = None\n", + " for _ in range(trials):\n", + " d, _ = problem_obj.random_design()\n", + " if len(problem_obj.check_constraints(d, cfg)) > 0:\n", + " continue\n", + " obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " if best_obj is None or objective_score(obj) < objective_score(best_obj):\n", + " best_obj = obj\n", + " if best_obj is None:\n", + " d, _ = problem_obj.random_design()\n", + " best_obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " return best_obj\n", + "\n", + "\n", + "def generate_design(generator_obj, cfg_vec: np.ndarray, lb: np.ndarray, ub: np.ndarray):\n", + " # cgan_1d Generator uses BatchNorm; switch to eval for single-sample inference.\n", + " was_training = generator_obj.training\n", + " generator_obj.eval()\n", + " try:\n", + " with th.no_grad():\n", + " c = th.tensor(cfg_vec[None, :], dtype=th.float32, device=device)\n", + " z = th.randn((1, LATENT_DIM), dtype=th.float32, device=device)\n", + " d = generator_obj(z, c).cpu().numpy()[0]\n", + " finally:\n", + " if was_training:\n", + " generator_obj.train()\n", + " return np.clip(d.astype(np.float32), lb, ub)\n", + "\n", + "\n", + "if RUN_OPTIONAL_SECTION:\n", + " lb = problem.design_space.low.astype(np.float32)\n", + " ub = problem.design_space.high.astype(np.float32)\n", + "\n", + " gen_objs = []\n", + " base_objs = []\n", + " feasible_count = 0\n", + "\n", + " for _ in range(EVAL_SAMPLES):\n", + " cfg = sample_condition_dict()\n", + " cfg_vec = cond_to_vec(cfg)\n", + "\n", + " gen_obj = None\n", + " for _retry in range(6):\n", + " d_gen = generate_design(generator, cfg_vec, lb, ub)\n", + " if len(problem.check_constraints(d_gen, cfg)) == 0:\n", + " gen_obj = problem.simulate(d_gen, {**cfg, **FAST_SIM_CFG})\n", + " feasible_count += 1\n", + " break\n", + " if gen_obj is None:\n", + " d_fallback, _ = problem.random_design()\n", + " gen_obj = problem.simulate(d_fallback, {**cfg, **FAST_SIM_CFG})\n", + "\n", + " base_obj = sample_baseline(problem, cfg, trials=8)\n", + " gen_objs.append(gen_obj)\n", + " base_objs.append(base_obj)\n", + "\n", + " gen_objs = np.stack(gen_objs)\n", + " base_objs = np.stack(base_objs)\n", + "\n", + " summary = {\n", + " \"generated_error_mean\": float(np.mean(gen_objs[:, 0])),\n", + " \"generated_energy_mean\": float(np.mean(gen_objs[:, 1])),\n", + " \"baseline_error_mean\": float(np.mean(base_objs[:, 0])),\n", + " \"baseline_energy_mean\": float(np.mean(base_objs[:, 1])),\n", + " \"generated_feasible_rate\": float(feasible_count / EVAL_SAMPLES),\n", + " }\n", + "\n", + " print(\"Optional extension summary (EngiOpt cgan_1d):\")\n", + " for k, v in summary.items():\n", + " print(f\" {k}: {v:.6f}\")\n", + "\n", + " np.savez(\n", + " OPTIONAL_ARTIFACT_DIR / \"optional_eval_summary_engiopt_cgan1d.npz\",\n", + " gen_objs=gen_objs,\n", + " base_objs=base_objs,\n", + " g_hist=np.array(g_hist, dtype=np.float32),\n", + " d_hist=np.array(d_hist, dtype=np.float32),\n", + " **summary,\n", + " )\n", + "\n", + " fig, axes = plt.subplots(1, 3, figsize=(14, 4))\n", + "\n", + " axes[0].plot(g_hist, label=\"g_loss\")\n", + " axes[0].plot(d_hist, label=\"d_loss\")\n", + " axes[0].set_title(\"EngiOpt cgan_1d training losses\")\n", + " axes[0].set_xlabel(\"epoch\")\n", + " axes[0].legend()\n", + " axes[0].grid(alpha=0.3)\n", + "\n", + " axes[1].hist(base_objs[:, 0], bins=12, alpha=0.6, label=\"baseline\")\n", + " axes[1].hist(gen_objs[:, 0], bins=12, alpha=0.6, label=\"generated\")\n", + " axes[1].set_title(\"Final tracking error\")\n", + " axes[1].set_xlabel(\"error [m]\")\n", + " axes[1].legend()\n", + "\n", + " axes[2].hist(base_objs[:, 1], bins=12, alpha=0.6, label=\"baseline\")\n", + " axes[2].hist(gen_objs[:, 1], bins=12, alpha=0.6, label=\"generated\")\n", + " axes[2].set_title(\"Actuation energy\")\n", + " axes[2].set_xlabel(\"energy [J]\")\n", + " axes[2].legend()\n", + "\n", + " fig.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"Skipped evaluation. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "markdown", + "id": "594df993", + "metadata": {}, + "source": [ + "### Discussion prompts for workshop synthesis\n", + "\n", + "1. How portable are EngiOpt models across domains with different design representations?\n", + "2. What is the minimum adapter contract needed to reuse a model on a new problem?\n", + "3. Should benchmark reporting require both feasibility and objective trade-off metrics for generated designs?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/requirements-colab.txt b/workshops/dcc26/requirements-colab.txt new file mode 100644 index 0000000..aee3142 --- /dev/null +++ b/workshops/dcc26/requirements-colab.txt @@ -0,0 +1,15 @@ +# Local convenience snapshot for workshop notebooks. +# Source of truth for Colab is the install/bootstrap cell inside each notebook. +# EngiOpt is intentionally installed from Git in notebook bootstrap cells. + +engibench[beams2d] +sqlitedict +matplotlib +seaborn +gymnasium +pybullet +tqdm +tyro +wandb +torch +torchvision diff --git a/workshops/dcc26/solutions/00_setup_api_warmup.ipynb b/workshops/dcc26/solutions/00_setup_api_warmup.ipynb new file mode 100644 index 0000000..d3d515e --- /dev/null +++ b/workshops/dcc26/solutions/00_setup_api_warmup.ipynb @@ -0,0 +1,302 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "eb3130a2", + "metadata": {}, + "source": [ + "# Notebook 00: Setup + API Warmup (DCC26)\n", + "\n", + "Reference walkthrough for inspecting EngiBench as a reproducible benchmark contract.\n" + ] + }, + { + "cell_type": "markdown", + "id": "5f88c0f0", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "95c5209f", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n" + ] + }, + { + "cell_type": "markdown", + "id": "bbec7887", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "Use this notebook as the baseline interpretation layer before any model training.\n", + "The objective is conceptual correctness, not speed.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f7c30d56", + "metadata": {}, + "source": [ + "## Why this warmup matters\n", + "\n", + "Most benchmarking disagreements come from interface misunderstandings, not algorithmic novelty.\n", + "This chapter removes that ambiguity up front.\n" + ] + }, + { + "cell_type": "markdown", + "id": "952d991e", + "metadata": {}, + "source": [ + "## Optional install cell (fresh Colab)\n", + "\n", + "Only required on fresh or reset Colab runtimes.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "946d0c0d", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"matplotlib\", \"seaborn\"]\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "002ea4f3", + "metadata": {}, + "source": [ + "### Step 1 - Initialize reproducible session\n", + "\n", + "Seed and version information define your execution context.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d44722b", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import engibench\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "SEED = 7\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)\n", + "\n", + "print(\"engibench version:\", engibench.__version__)\n", + "print(\"seed:\", SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "8049d01e", + "metadata": {}, + "source": [ + "### Step 2 - Instantiate benchmark problem\n", + "\n", + "Inspect problem metadata and ensure the design/condition/objective contract is explicit.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62611bc5", + "metadata": {}, + "outputs": [], + "source": [ + "problem = Beams2D(seed=SEED)\n", + "\n", + "print(\"Problem class:\", type(problem).__name__)\n", + "print(\"Design space:\", problem.design_space)\n", + "print(\"Objectives:\", problem.objectives)\n", + "print(\"Conditions instance:\", problem.conditions)\n", + "print(\"Condition keys:\", problem.conditions_keys)\n", + "print(\"Dataset ID:\", problem.dataset_id)" + ] + }, + { + "cell_type": "markdown", + "id": "7a0b4e5d", + "metadata": {}, + "source": [ + "### Step 3 - Inspect dataset structure\n", + "\n", + "Confirm split semantics and field consistency before using this data in modeling notebooks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d990d6f3", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = problem.dataset\n", + "print(dataset)\n", + "\n", + "sample_idx = 0\n", + "design = np.array(dataset[\"train\"][\"optimal_design\"][sample_idx])\n", + "config = {k: dataset[\"train\"][k][sample_idx] for k in problem.conditions_keys}\n", + "\n", + "print(\"Sample design shape:\", design.shape)\n", + "print(\"Sample config:\", config)" + ] + }, + { + "cell_type": "markdown", + "id": "5dd9c588", + "metadata": {}, + "source": [ + "### Step 4 - Visualize one benchmark design\n", + "\n", + "Use rendering to align numerical representation with engineering intuition.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3935f407", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = problem.render(design)\n", + "ax.set_title(\"Sample Beams2D design from training split\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "772416a2", + "metadata": {}, + "source": [ + "### Step 5 - Test constraint semantics\n", + "\n", + "A controlled violation case clarifies what `check_constraints` is actually diagnosing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe14de33", + "metadata": {}, + "outputs": [], + "source": [ + "# One explicit constraint check with intentionally mismatched volume fraction\n", + "bad_config = dict(config)\n", + "bad_config[\"volfrac\"] = 0.2\n", + "violations = problem.check_constraints(design=design, config=bad_config)\n", + "\n", + "print(\"Violation count:\", len(violations))\n", + "if violations:\n", + " print(violations)\n", + "else:\n", + " print(\"No violations found\")" + ] + }, + { + "cell_type": "markdown", + "id": "bec1e57f", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "a37bb417", + "metadata": {}, + "source": [ + "## Next\n", + "\n", + "Continue with Notebook 01 for model integration and artifact generation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "6fe3f1a4", + "metadata": {}, + "source": [ + "## Reflection prompts\n", + "\n", + "- What would make two reported results incomparable on this benchmark?\n", + "- Which configuration fields are non-negotiable in method reporting?\n" + ] + }, + { + "cell_type": "markdown", + "id": "547b3933", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/solutions/01_train_generate.ipynb b/workshops/dcc26/solutions/01_train_generate.ipynb new file mode 100644 index 0000000..8750768 --- /dev/null +++ b/workshops/dcc26/solutions/01_train_generate.ipynb @@ -0,0 +1,598 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5ec86a66", + "metadata": {}, + "source": [ + "# Notebook 01: Train + Generate with EngiOpt CGAN-2D (DCC26)\n", + "\n", + "Reference implementation for method integration, diagnostics, and artifact contract creation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "28e779fa", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ca81b1b9", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n" + ] + }, + { + "cell_type": "markdown", + "id": "52364318", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "This notebook is designed to be publication-grade reproducibility scaffolding:\n", + "clear controls, explicit artifacts, and interpretable training diagnostics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6bc375a", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"sqlitedict\", \"matplotlib\", \"tqdm\", \"tyro\", \"wandb\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "1f65b99f", + "metadata": {}, + "source": [ + "## Part A: Configuration and runtime controls\n", + "\n", + "Establish deterministic setup and artifact policy before touching model code.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f0e9f4a5", + "metadata": {}, + "source": [ + "### EngiBench vs EngiOpt roles in this notebook\n", + "\n", + "EngiBench provides the benchmark contract; EngiOpt provides the method implementation.\n", + "Conflating these layers is a common source of irreproducible claims.\n" + ] + }, + { + "cell_type": "markdown", + "id": "7ede774c", + "metadata": {}, + "source": [ + "### Step 1 - Configure reproducible environment\n", + "\n", + "Seed control and path control are first-class experimental settings, not boilerplate.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d947da0", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import random\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "try:\n", + " from engiopt.cgan_2d.cgan_2d import Generator as EngiOptCGAN2DGenerator\n", + "except ModuleNotFoundError as exc:\n", + " raise ModuleNotFoundError(\n", + " \"Could not import engiopt model class. Run the bootstrap cell first; on Colab, restart runtime after install if needed.\"\n", + " ) from exc\n", + "\n", + "# Optional W&B integration\n", + "USE_WANDB_ARTIFACTS = False\n", + "WANDB_PROJECT = \"dcc26-workshop\"\n", + "WANDB_ENTITY = None\n", + "WANDB_ARTIFACT_NAME = \"dcc26_beams2d_generated_artifacts\"\n", + "WANDB_ARTIFACT_ALIAS = \"latest\"\n", + "WANDB_LOG_TRAINING = True\n", + "\n", + "\n", + "def resolve_artifact_dir(create: bool = False) -> Path:\n", + " in_colab = \"google.colab\" in sys.modules\n", + " path = Path(\"/content/dcc26_artifacts\") if in_colab else Path(\"workshops/dcc26/artifacts\")\n", + " if create:\n", + " path.mkdir(parents=True, exist_ok=True)\n", + " return path\n", + "\n", + "\n", + "SEED = 7\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)\n", + "th.manual_seed(SEED)\n", + "if th.cuda.is_available():\n", + " th.cuda.manual_seed_all(SEED)\n", + "\n", + "DEVICE = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + "print(\"device:\", DEVICE)\n", + "\n", + "ARTIFACT_DIR = resolve_artifact_dir(create=True)\n", + "print(\"artifact dir:\", ARTIFACT_DIR)\n", + "\n", + "CKPT_PATH = ARTIFACT_DIR / \"engiopt_cgan2d_generator_supervised.pt\"\n", + "HISTORY_PATH = ARTIFACT_DIR / \"training_history.csv\"\n", + "TRAIN_CURVE_PATH = ARTIFACT_DIR / \"training_curve.png\"\n", + "LATENT_DIM = 32" + ] + }, + { + "cell_type": "markdown", + "id": "acc18dbf", + "metadata": {}, + "source": [ + "## Part B: Load Beams2D data and build a stable workshop subset\n", + "\n", + "We intentionally constrain runtime while preserving the full benchmark interaction pattern.\n" + ] + }, + { + "cell_type": "markdown", + "id": "281ef1bd", + "metadata": {}, + "source": [ + "### Training diagnostics to monitor\n", + "\n", + "Track trend shape (stability/collapse), not only endpoint value.\n", + "Diagnostics are evidence, not decoration.\n" + ] + }, + { + "cell_type": "markdown", + "id": "0ca03295", + "metadata": {}, + "source": [ + "### Step 2 - Build training slice from EngiBench dataset\n", + "\n", + "Create aligned condition/design tensors and document scaling assumptions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "684c4a1f", + "metadata": {}, + "outputs": [], + "source": [ + "problem = Beams2D(seed=SEED)\n", + "train_ds = problem.dataset[\"train\"]\n", + "test_ds = problem.dataset[\"test\"]\n", + "\n", + "condition_keys = problem.conditions_keys\n", + "print(\"condition keys:\", condition_keys)\n", + "\n", + "N_TRAIN = 512\n", + "subset_idx = np.random.default_rng(SEED).choice(len(train_ds), size=N_TRAIN, replace=False)\n", + "\n", + "conds_np = np.stack([np.array(train_ds[k])[subset_idx].astype(np.float32) for k in condition_keys], axis=1)\n", + "designs_np = np.array(train_ds[\"optimal_design\"])[subset_idx].astype(np.float32)\n", + "\n", + "# EngiOpt CGAN generator emits tanh-scaled outputs in [-1, 1].\n", + "targets_np = (designs_np * 2.0) - 1.0\n", + "\n", + "print(\"conditions shape:\", conds_np.shape)\n", + "print(\"designs shape:\", designs_np.shape)\n", + "print(\"target range:\", float(targets_np.min()), \"to\", float(targets_np.max()))" + ] + }, + { + "cell_type": "markdown", + "id": "f4aec55d", + "metadata": {}, + "source": [ + "### Step 3 - Define EngiOpt model and optimization objects\n", + "\n", + "Model dimensions should derive from benchmark metadata, not hard-coded guesswork.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c5a8fdf", + "metadata": {}, + "outputs": [], + "source": [ + "model = EngiOptCGAN2DGenerator(\n", + " latent_dim=LATENT_DIM,\n", + " n_conds=conds_np.shape[1],\n", + " design_shape=problem.design_space.shape,\n", + ").to(DEVICE)\n", + "\n", + "optimizer = th.optim.Adam(model.parameters(), lr=1e-3)\n", + "criterion = nn.MSELoss()\n", + "\n", + "\n", + "def sample_noise(batch_size: int) -> th.Tensor:\n", + " return th.randn((batch_size, LATENT_DIM), device=DEVICE, dtype=th.float32)" + ] + }, + { + "cell_type": "markdown", + "id": "f796eff4", + "metadata": {}, + "source": [ + "### Step 4 - Train (or load) with diagnostics\n", + "\n", + "Persist checkpoint and training traces so downstream evaluation can be audited and repeated.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b09e4d3", + "metadata": {}, + "outputs": [], + "source": [ + "TRAIN_FROM_SCRATCH = True\n", + "# Train on compact subset for workshop-time reliability (not full benchmark SOTA mode).\n", + "EPOCHS = 8\n", + "BATCH_SIZE = 64\n", + "\n", + "train_losses = []\n", + "wandb_train_run = None\n", + "\n", + "if TRAIN_FROM_SCRATCH:\n", + " ds = TensorDataset(th.tensor(conds_np), th.tensor(targets_np))\n", + " dl = DataLoader(ds, batch_size=BATCH_SIZE, shuffle=True)\n", + "\n", + " if USE_WANDB_ARTIFACTS and WANDB_LOG_TRAINING:\n", + " import wandb\n", + "\n", + " wandb_train_run = wandb.init(\n", + " project=WANDB_PROJECT,\n", + " entity=WANDB_ENTITY,\n", + " job_type=\"train\",\n", + " config={\n", + " \"seed\": SEED,\n", + " \"epochs\": EPOCHS,\n", + " \"batch_size\": BATCH_SIZE,\n", + " \"n_train\": int(N_TRAIN),\n", + " \"latent_dim\": LATENT_DIM,\n", + " \"model_family\": \"engiopt.cgan_2d.Generator\",\n", + " },\n", + " reinit=True,\n", + " )\n", + "\n", + " for epoch in range(EPOCHS):\n", + " # Epoch-average loss is what we compare across quick workshop runs.\n", + " model.train()\n", + " epoch_loss = 0.0\n", + "\n", + " for cond_batch, target_batch in dl:\n", + " cond_batch = cond_batch.to(DEVICE)\n", + " target_batch = target_batch.to(DEVICE)\n", + "\n", + " pred = model(sample_noise(cond_batch.shape[0]), cond_batch)\n", + " loss = criterion(pred, target_batch)\n", + "\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " epoch_loss += float(loss.item())\n", + "\n", + " epoch_avg = epoch_loss / len(dl)\n", + " train_losses.append(epoch_avg)\n", + " print(f\"epoch {epoch + 1:02d}/{EPOCHS} - loss: {epoch_avg:.4f}\")\n", + "\n", + " if wandb_train_run is not None:\n", + " wandb_train_run.log({\"train/loss\": epoch_avg, \"epoch\": epoch + 1})\n", + "\n", + " th.save(\n", + " {\n", + " \"model\": model.state_dict(),\n", + " \"condition_keys\": condition_keys,\n", + " \"latent_dim\": LATENT_DIM,\n", + " \"model_family\": \"engiopt.cgan_2d.Generator\",\n", + " },\n", + " CKPT_PATH,\n", + " )\n", + " print(\"saved checkpoint to\", CKPT_PATH)\n", + "\n", + " history_df = pd.DataFrame({\"epoch\": np.arange(1, len(train_losses) + 1), \"train_loss\": train_losses})\n", + " history_df.to_csv(HISTORY_PATH, index=False)\n", + "\n", + " fig, ax = plt.subplots(figsize=(6, 3.5))\n", + " ax.plot(history_df[\"epoch\"], history_df[\"train_loss\"], marker=\"o\")\n", + " ax.set_xlabel(\"Epoch\")\n", + " ax.set_ylabel(\"MSE loss\")\n", + " ax.set_title(\"Notebook 01 training curve\")\n", + " ax.grid(alpha=0.3)\n", + " fig.tight_layout()\n", + " fig.savefig(TRAIN_CURVE_PATH, dpi=150)\n", + " plt.show()\n", + "\n", + " if wandb_train_run is not None:\n", + " import wandb\n", + "\n", + " wandb_train_run.log({\"train/loss_curve\": wandb.Image(str(TRAIN_CURVE_PATH))})\n", + " wandb_train_run.finish()\n", + "elif CKPT_PATH.exists():\n", + " ckpt = th.load(CKPT_PATH, map_location=DEVICE)\n", + " model.load_state_dict(ckpt[\"model\"])\n", + " model.eval()\n", + " print(\"loaded checkpoint from\", CKPT_PATH)\n", + "\n", + " if HISTORY_PATH.exists():\n", + " history_df = pd.read_csv(HISTORY_PATH)\n", + " else:\n", + " history_df = pd.DataFrame(columns=[\"epoch\", \"train_loss\"])\n", + "else:\n", + " raise FileNotFoundError(f\"No checkpoint found at {CKPT_PATH}. Set TRAIN_FROM_SCRATCH=True or provide a checkpoint.\")" + ] + }, + { + "cell_type": "markdown", + "id": "9c46c7fb", + "metadata": {}, + "source": [ + "## Part C: Condition-driven generation and quick sanity checks\n", + "\n", + "Generate candidates for held-out conditions and run quick feasibility checks before full simulation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "22cf9ccc", + "metadata": {}, + "source": [ + "### Scientific checkpoint before Notebook 02\n", + "\n", + "Ask whether outputs are merely plausible-looking or genuinely evaluation-ready.\n", + "Notebook 02 will resolve this quantitatively.\n" + ] + }, + { + "cell_type": "markdown", + "id": "91f5c0e3", + "metadata": {}, + "source": [ + "### Step 5 - Generate conditioned designs\n", + "\n", + "Use consistent test sampling so comparisons remain interpretable across runs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "615dbe0e", + "metadata": {}, + "outputs": [], + "source": [ + "rng = np.random.default_rng(SEED)\n", + "N_SAMPLES = 24\n", + "selected = rng.choice(len(test_ds), size=N_SAMPLES, replace=False)\n", + "\n", + "test_conds = np.stack([np.array(test_ds[k])[selected].astype(np.float32) for k in condition_keys], axis=1)\n", + "baseline_designs = np.array(test_ds[\"optimal_design\"])[selected].astype(np.float32)\n", + "\n", + "model.eval()\n", + "with th.no_grad():\n", + " tanh_out = model(sample_noise(N_SAMPLES), th.tensor(test_conds, device=DEVICE))\n", + " gen_designs_t = ((tanh_out.clamp(-1.0, 1.0) + 1.0) / 2.0).clamp(0.0, 1.0)\n", + "gen_designs = gen_designs_t.detach().cpu().numpy().astype(np.float32)\n", + "\n", + "conditions_records = []\n", + "for i in range(N_SAMPLES):\n", + " rec = {}\n", + " for j, k in enumerate(condition_keys):\n", + " v = test_conds[i, j]\n", + " rec[k] = bool(v) if k == \"overhang_constraint\" else float(v)\n", + " conditions_records.append(rec)\n", + "\n", + "# Quick checks before full Notebook 02 evaluation\n", + "viol_ratio = np.mean(\n", + " [\n", + " len(problem.check_constraints(design=d, config=cfg)) > 0\n", + " for d, cfg in zip(gen_designs, conditions_records, strict=True)\n", + " ]\n", + ")\n", + "print(\"generated shape:\", gen_designs.shape)\n", + "print(\"baseline shape:\", baseline_designs.shape)\n", + "print(\"generated violation ratio (quick check):\", float(viol_ratio))" + ] + }, + { + "cell_type": "markdown", + "id": "ccf7721d", + "metadata": {}, + "source": [ + "### Step 6 - Export artifact contract\n", + "\n", + "These files are the reproducible boundary between modeling and evaluation notebooks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcb64597", + "metadata": {}, + "outputs": [], + "source": [ + "# Artifact contract consumed by Notebook 02 and optional W&B logging.\n", + "generated_path = ARTIFACT_DIR / \"generated_designs.npy\"\n", + "baseline_path = ARTIFACT_DIR / \"baseline_designs.npy\"\n", + "conditions_path = ARTIFACT_DIR / \"conditions.json\"\n", + "\n", + "np.save(generated_path, gen_designs)\n", + "np.save(baseline_path, baseline_designs)\n", + "with open(conditions_path, \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(conditions_records, f, indent=2)\n", + "\n", + "print(\"Saved artifacts to\", ARTIFACT_DIR)\n", + "print(\"-\", generated_path)\n", + "print(\"-\", baseline_path)\n", + "print(\"-\", conditions_path)\n", + "print(\"-\", CKPT_PATH)\n", + "print(\"-\", HISTORY_PATH)\n", + "print(\"-\", TRAIN_CURVE_PATH)\n", + "\n", + "if USE_WANDB_ARTIFACTS:\n", + " try:\n", + " import wandb\n", + "\n", + " run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, job_type=\"artifact-upload\", reinit=True)\n", + " run.log(\n", + " {\n", + " \"artifact/generated_mean_density\": float(gen_designs.mean()),\n", + " \"artifact/generated_std_density\": float(gen_designs.std()),\n", + " }\n", + " )\n", + "\n", + " artifact = wandb.Artifact(\n", + " WANDB_ARTIFACT_NAME,\n", + " type=\"dataset\",\n", + " description=\"DCC26 Notebook 01 artifacts: arrays, conditions, checkpoint, and training diagnostics.\",\n", + " )\n", + " for path in [generated_path, baseline_path, conditions_path, CKPT_PATH, HISTORY_PATH, TRAIN_CURVE_PATH]:\n", + " if path.exists():\n", + " artifact.add_file(str(path))\n", + " run.log_artifact(artifact, aliases=[WANDB_ARTIFACT_ALIAS])\n", + " run.finish()\n", + " print(\"Uploaded artifacts to W&B:\", WANDB_ARTIFACT_NAME)\n", + " except Exception as exc:\n", + " print(\"W&B upload failed (continuing with local artifacts only):\", exc)" + ] + }, + { + "cell_type": "markdown", + "id": "17f27f2d", + "metadata": {}, + "source": [ + "### Step 7 - Quick visual QA\n", + "\n", + "Visual checks detect obvious failure patterns early (blank/noise/mode collapse).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3cc5a2bd", + "metadata": {}, + "outputs": [], + "source": [ + "# Visual side-by-side snapshot (generated vs baseline)\n", + "fig, axes = plt.subplots(2, 6, figsize=(14, 5))\n", + "for i in range(6):\n", + " axes[0, i].imshow(gen_designs[i], cmap=\"gray\", vmin=0, vmax=1)\n", + " axes[0, i].set_title(f\"gen {i}\")\n", + " axes[0, i].axis(\"off\")\n", + "\n", + " axes[1, i].imshow(baseline_designs[i], cmap=\"gray\", vmin=0, vmax=1)\n", + " axes[1, i].set_title(f\"base {i}\")\n", + " axes[1, i].axis(\"off\")\n", + "\n", + "fig.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "86f2eec2", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "3005644d", + "metadata": {}, + "source": [ + "## Next\n", + "\n", + "Continue with Notebook 02 for benchmark-grade evaluation and reporting outputs.\n" + ] + }, + { + "cell_type": "markdown", + "id": "33f6b236", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/solutions/02_evaluate_metrics.ipynb b/workshops/dcc26/solutions/02_evaluate_metrics.ipynb new file mode 100644 index 0000000..4122708 --- /dev/null +++ b/workshops/dcc26/solutions/02_evaluate_metrics.ipynb @@ -0,0 +1,612 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a4862a7e", + "metadata": {}, + "source": [ + "# Notebook 02: Evaluation + Metrics (DCC26)\n", + "\n", + "Reference implementation for benchmark-grade evaluation and result interpretation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "1daf74ad", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "5becd891", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n" + ] + }, + { + "cell_type": "markdown", + "id": "fb72fd19", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "This notebook operationalizes rigorous comparison: same conditions, same simulator, explicit metrics, reproducible exports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f496a538", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"sqlitedict\", \"matplotlib\", \"tqdm\", \"tyro\", \"wandb\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "685cf46f", + "metadata": {}, + "source": [ + "## Artifact loading strategy\n", + "\n", + "Resolution order is explicit to avoid hidden state:\n", + "local artifacts -> optional W&B pull -> local auto-build fallback.\n" + ] + }, + { + "cell_type": "markdown", + "id": "34f48177", + "metadata": {}, + "source": [ + "### Why these metrics matter for benchmarking\n", + "\n", + "Objective-only reporting can hide infeasibility or mode collapse.\n", + "The metric suite is intentionally multi-dimensional.\n" + ] + }, + { + "cell_type": "markdown", + "id": "14103ff3", + "metadata": {}, + "source": [ + "### Step 1 - Resolve artifact source and recovery path\n", + "\n", + "Validate artifact integrity before evaluation; otherwise downstream metrics are misleading.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f7f1979", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import random\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "from engibench.problems.beams2d.v0 import Beams2D\n", + "\n", + "try:\n", + " from engiopt.cgan_2d.cgan_2d import Generator as EngiOptCGAN2DGenerator\n", + "except ModuleNotFoundError as exc:\n", + " raise ModuleNotFoundError(\n", + " \"Could not import engiopt model class. Run the bootstrap cell first; on Colab, restart runtime after install if needed.\"\n", + " ) from exc\n", + "\n", + "USE_WANDB_ARTIFACTS = False\n", + "WANDB_PROJECT = \"dcc26-workshop\"\n", + "WANDB_ENTITY = None\n", + "WANDB_ARTIFACT_NAME = \"dcc26_beams2d_generated_artifacts\"\n", + "WANDB_ARTIFACT_ALIAS = \"latest\"\n", + "\n", + "# Self-heal path for workshop robustness\n", + "AUTO_BUILD_ARTIFACTS_IF_MISSING = True\n", + "\n", + "\n", + "def resolve_artifact_dir(create: bool = False) -> Path:\n", + " in_colab = \"google.colab\" in sys.modules\n", + " path = Path(\"/content/dcc26_artifacts\") if in_colab else Path(\"workshops/dcc26/artifacts\")\n", + " if create:\n", + " path.mkdir(parents=True, exist_ok=True)\n", + " return path\n", + "\n", + "\n", + "def build_artifacts_locally(\n", + " artifact_dir: Path,\n", + " seed: int = 7,\n", + " n_train: int = 512,\n", + " n_samples: int = 24,\n", + " epochs: int = 8,\n", + " batch_size: int = 64,\n", + " latent_dim: int = 32,\n", + ") -> None:\n", + " print(\"Building Notebook 01-style artifacts locally with EngiOpt...\")\n", + "\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " th.manual_seed(seed)\n", + " if th.cuda.is_available():\n", + " th.cuda.manual_seed_all(seed)\n", + "\n", + " device = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + " problem = Beams2D(seed=seed)\n", + " train_ds = problem.dataset[\"train\"]\n", + " test_ds = problem.dataset[\"test\"]\n", + " condition_keys = problem.conditions_keys\n", + "\n", + " rng = np.random.default_rng(seed)\n", + " subset_size = min(n_train, len(train_ds))\n", + " subset_idx = rng.choice(len(train_ds), size=subset_size, replace=False)\n", + "\n", + " conds_np = np.stack([np.array(train_ds[k])[subset_idx].astype(np.float32) for k in condition_keys], axis=1)\n", + " designs_np = np.array(train_ds[\"optimal_design\"])[subset_idx].astype(np.float32)\n", + " targets_np = designs_np * 2.0 - 1.0\n", + "\n", + " model = EngiOptCGAN2DGenerator(\n", + " latent_dim=latent_dim,\n", + " n_conds=conds_np.shape[1],\n", + " design_shape=problem.design_space.shape,\n", + " ).to(device)\n", + " optimizer = th.optim.Adam(model.parameters(), lr=1e-3)\n", + " criterion = nn.MSELoss()\n", + "\n", + " def sample_noise(batch: int) -> th.Tensor:\n", + " return th.randn((batch, latent_dim), device=device, dtype=th.float32)\n", + "\n", + " ds = TensorDataset(th.tensor(conds_np), th.tensor(targets_np))\n", + " dl = DataLoader(ds, batch_size=batch_size, shuffle=True)\n", + "\n", + " train_losses = []\n", + " for epoch in range(epochs):\n", + " model.train()\n", + " epoch_loss = 0.0\n", + " for cond_batch, target_batch in dl:\n", + " cond_batch = cond_batch.to(device)\n", + " target_batch = target_batch.to(device)\n", + "\n", + " pred = model(sample_noise(cond_batch.shape[0]), cond_batch)\n", + " loss = criterion(pred, target_batch)\n", + "\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " epoch_loss += float(loss.item())\n", + "\n", + " epoch_avg = epoch_loss / len(dl)\n", + " train_losses.append(epoch_avg)\n", + " print(f\"bootstrap epoch {epoch + 1:02d}/{epochs} - loss: {epoch_avg:.4f}\")\n", + "\n", + " sample_count = min(n_samples, len(test_ds))\n", + " selected = rng.choice(len(test_ds), size=sample_count, replace=False)\n", + " test_conds = np.stack([np.array(test_ds[k])[selected].astype(np.float32) for k in condition_keys], axis=1)\n", + " baseline_designs = np.array(test_ds[\"optimal_design\"])[selected].astype(np.float32)\n", + "\n", + " model.eval()\n", + " with th.no_grad():\n", + " tanh_out = model(sample_noise(sample_count), th.tensor(test_conds, device=device))\n", + " gen_designs_t = ((tanh_out.clamp(-1.0, 1.0) + 1.0) / 2.0).clamp(0.0, 1.0)\n", + " gen_designs = gen_designs_t.detach().cpu().numpy().astype(np.float32)\n", + "\n", + " conditions_records = []\n", + " for i in range(sample_count):\n", + " rec = {}\n", + " for j, k in enumerate(condition_keys):\n", + " v = test_conds[i, j]\n", + " rec[k] = bool(v) if k == \"overhang_constraint\" else float(v)\n", + " conditions_records.append(rec)\n", + "\n", + " artifact_dir.mkdir(parents=True, exist_ok=True)\n", + " np.save(artifact_dir / \"generated_designs.npy\", gen_designs)\n", + " np.save(artifact_dir / \"baseline_designs.npy\", baseline_designs)\n", + " with open(artifact_dir / \"conditions.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(conditions_records, f, indent=2)\n", + "\n", + " pd.DataFrame({\"epoch\": np.arange(1, len(train_losses) + 1), \"train_loss\": train_losses}).to_csv(\n", + " artifact_dir / \"training_history.csv\", index=False\n", + " )\n", + "\n", + " th.save(\n", + " {\n", + " \"model\": model.state_dict(),\n", + " \"condition_keys\": condition_keys,\n", + " \"latent_dim\": latent_dim,\n", + " \"model_family\": \"engiopt.cgan_2d.Generator\",\n", + " },\n", + " artifact_dir / \"engiopt_cgan2d_generator_supervised.pt\",\n", + " )\n", + "\n", + " print(\"Built artifacts at\", artifact_dir)\n", + "\n", + "\n", + "ARTIFACT_DIR = resolve_artifact_dir(create=True)\n", + "required = [\n", + " ARTIFACT_DIR / \"generated_designs.npy\",\n", + " ARTIFACT_DIR / \"baseline_designs.npy\",\n", + " ARTIFACT_DIR / \"conditions.json\",\n", + "]\n", + "\n", + "if not all(p.exists() for p in required):\n", + " if USE_WANDB_ARTIFACTS:\n", + " try:\n", + " import wandb\n", + "\n", + " run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, job_type=\"artifact-download\", reinit=True)\n", + " if WANDB_ENTITY:\n", + " artifact_ref = f\"{WANDB_ENTITY}/{WANDB_PROJECT}/{WANDB_ARTIFACT_NAME}:{WANDB_ARTIFACT_ALIAS}\"\n", + " else:\n", + " artifact_ref = f\"{WANDB_PROJECT}/{WANDB_ARTIFACT_NAME}:{WANDB_ARTIFACT_ALIAS}\"\n", + " artifact = run.use_artifact(artifact_ref, type=\"dataset\")\n", + " artifact.download(root=str(ARTIFACT_DIR))\n", + " run.finish()\n", + " print(\"Downloaded artifacts from W&B to\", ARTIFACT_DIR)\n", + " except Exception as exc:\n", + " if AUTO_BUILD_ARTIFACTS_IF_MISSING:\n", + " print(\"W&B download failed; switching to local artifact build:\", exc)\n", + " build_artifacts_locally(ARTIFACT_DIR)\n", + " else:\n", + " raise FileNotFoundError(\n", + " \"Artifacts missing locally and W&B download failed. \"\n", + " \"Run Notebook 01 first or disable USE_WANDB_ARTIFACTS. \"\n", + " f\"Details: {exc}\"\n", + " ) from exc\n", + " elif AUTO_BUILD_ARTIFACTS_IF_MISSING:\n", + " build_artifacts_locally(ARTIFACT_DIR)\n", + " else:\n", + " missing = \"\\n\".join(f\"- {p}\" for p in required if not p.exists())\n", + " raise FileNotFoundError(\n", + " \"Notebook 01 artifacts not found. Run Notebook 01 first (including export cell), \"\n", + " \"or enable USE_WANDB_ARTIFACTS to fetch from W&B. Missing files:\\\\n\" + missing\n", + " )\n", + "\n", + "print(\"using artifact dir:\", ARTIFACT_DIR)\n", + "\n", + "gen_designs = np.load(ARTIFACT_DIR / \"generated_designs.npy\")\n", + "baseline_designs = np.load(ARTIFACT_DIR / \"baseline_designs.npy\")\n", + "with open(ARTIFACT_DIR / \"conditions.json\", encoding=\"utf-8\") as f:\n", + " conditions = json.load(f)\n", + "\n", + "print(\"generated:\", gen_designs.shape)\n", + "print(\"baseline:\", baseline_designs.shape)\n", + "print(\"conditions:\", len(conditions))" + ] + }, + { + "cell_type": "markdown", + "id": "8e6bf426", + "metadata": {}, + "source": [ + "## Per-sample evaluation loop\n", + "\n", + "Evaluate generated and baseline designs sample-by-sample under identical simulator resets.\n" + ] + }, + { + "cell_type": "markdown", + "id": "57092416", + "metadata": {}, + "source": [ + "### Step 2 - Run per-sample physics evaluation\n", + "\n", + "Constraint and objective are computed separately to expose failure modes clearly.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "439ed8ad", + "metadata": {}, + "outputs": [], + "source": [ + "# Constraint and simulation evaluations are decoupled to diagnose failure modes clearly.\n", + "problem = Beams2D(seed=7)\n", + "\n", + "rows = []\n", + "for i, (g, b, cfg) in enumerate(zip(gen_designs, baseline_designs, conditions, strict=True)):\n", + " g_viol = problem.check_constraints(design=g, config=cfg)\n", + " b_viol = problem.check_constraints(design=b, config=cfg)\n", + "\n", + " # Reset before simulator calls for reproducible comparisons.\n", + " problem.reset(seed=7)\n", + " g_obj = float(problem.simulate(g, config=cfg)[0])\n", + " problem.reset(seed=7)\n", + " b_obj = float(problem.simulate(b, config=cfg)[0])\n", + "\n", + " rows.append(\n", + " {\n", + " \"sample\": i,\n", + " \"gen_obj\": g_obj,\n", + " \"base_obj\": b_obj,\n", + " \"gen_minus_base\": g_obj - b_obj,\n", + " \"gen_violations\": len(g_viol),\n", + " \"base_violations\": len(b_viol),\n", + " }\n", + " )\n", + "\n", + "results = pd.DataFrame(rows)\n", + "results.head()" + ] + }, + { + "cell_type": "markdown", + "id": "b9bd54e5", + "metadata": {}, + "source": [ + "### Step 3 - Compute benchmark metrics\n", + "\n", + "Summaries should support both leaderboard-style comparison and scientific diagnosis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1aed4828", + "metadata": {}, + "outputs": [], + "source": [ + "# Diversity proxy (intra-generated spread) and novelty proxy (distance to train set).\n", + "def mean_pairwise_l2(designs: np.ndarray) -> float:\n", + " flat = designs.reshape(designs.shape[0], -1)\n", + " n = flat.shape[0]\n", + " if n < 2:\n", + " return 0.0\n", + " dists = []\n", + " for i in range(n):\n", + " for j in range(i + 1, n):\n", + " dists.append(float(np.linalg.norm(flat[i] - flat[j])))\n", + " return float(np.mean(dists))\n", + "\n", + "\n", + "def mean_nn_distance_to_reference(designs: np.ndarray, reference_designs: np.ndarray) -> float:\n", + " q = designs.reshape(designs.shape[0], -1)\n", + " r = reference_designs.reshape(reference_designs.shape[0], -1)\n", + " nn_dists = []\n", + " for i in range(q.shape[0]):\n", + " d = np.linalg.norm(r - q[i][None, :], axis=1)\n", + " nn_dists.append(float(np.min(d)))\n", + " return float(np.mean(nn_dists))\n", + "\n", + "\n", + "train_designs_full = np.array(problem.dataset[\"train\"][\"optimal_design\"]).astype(np.float32)\n", + "ref_idx = np.random.default_rng(7).choice(len(train_designs_full), size=min(1024, len(train_designs_full)), replace=False)\n", + "train_reference = train_designs_full[ref_idx]\n", + "\n", + "summary = {\n", + " \"n_samples\": int(len(results)),\n", + " \"gen_obj_mean\": float(results[\"gen_obj\"].mean()),\n", + " \"base_obj_mean\": float(results[\"base_obj\"].mean()),\n", + " \"objective_gap_mean\": float(results[\"gen_minus_base\"].mean()),\n", + " \"improvement_rate\": float((results[\"gen_obj\"] < results[\"base_obj\"]).mean()),\n", + " \"gen_violation_ratio\": float((results[\"gen_violations\"] > 0).mean()),\n", + " \"base_violation_ratio\": float((results[\"base_violations\"] > 0).mean()),\n", + " \"gen_feasible_rate\": float((results[\"gen_violations\"] == 0).mean()),\n", + " \"gen_diversity_l2\": mean_pairwise_l2(gen_designs),\n", + " \"gen_novelty_to_train_l2\": mean_nn_distance_to_reference(gen_designs, train_reference),\n", + "}\n", + "\n", + "summary_df = pd.DataFrame([summary])\n", + "summary_df" + ] + }, + { + "cell_type": "markdown", + "id": "2c900998", + "metadata": {}, + "source": [ + "### Step 4 - Export evidence artifacts\n", + "\n", + "Persist tables/plots and optional W&B logs so external reviewers can inspect claims.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88ab86bc", + "metadata": {}, + "outputs": [], + "source": [ + "results_path = ARTIFACT_DIR / \"per_sample_metrics.csv\"\n", + "summary_path = ARTIFACT_DIR / \"metrics_summary.csv\"\n", + "hist_path = ARTIFACT_DIR / \"objective_histogram.png\"\n", + "grid_path = ARTIFACT_DIR / \"design_grid.png\"\n", + "scatter_path = ARTIFACT_DIR / \"objective_scatter.png\"\n", + "\n", + "results.to_csv(results_path, index=False)\n", + "summary_df.to_csv(summary_path, index=False)\n", + "\n", + "fig, ax = plt.subplots(figsize=(7, 4))\n", + "ax.hist(results[\"gen_obj\"], bins=10, alpha=0.7, label=\"generated\")\n", + "ax.hist(results[\"base_obj\"], bins=10, alpha=0.7, label=\"baseline\")\n", + "ax.set_xlabel(\"Compliance objective (lower is better)\")\n", + "ax.set_ylabel(\"Count\")\n", + "ax.set_title(\"Generated vs baseline objective distribution\")\n", + "ax.legend()\n", + "fig.tight_layout()\n", + "fig.savefig(hist_path, dpi=150)\n", + "plt.show()\n", + "\n", + "fig2, ax2 = plt.subplots(figsize=(5, 5))\n", + "ax2.scatter(results[\"base_obj\"], results[\"gen_obj\"], alpha=0.8)\n", + "min_v = min(results[\"base_obj\"].min(), results[\"gen_obj\"].min())\n", + "max_v = max(results[\"base_obj\"].max(), results[\"gen_obj\"].max())\n", + "ax2.plot([min_v, max_v], [min_v, max_v], \"--\", color=\"black\", linewidth=1)\n", + "ax2.set_xlabel(\"Baseline objective\")\n", + "ax2.set_ylabel(\"Generated objective\")\n", + "ax2.set_title(\"Per-sample objective comparison\")\n", + "fig2.tight_layout()\n", + "fig2.savefig(scatter_path, dpi=150)\n", + "plt.show()\n", + "\n", + "print(\"Saved:\")\n", + "print(\"-\", results_path)\n", + "print(\"-\", summary_path)\n", + "print(\"-\", hist_path)\n", + "print(\"-\", scatter_path)\n", + "\n", + "if USE_WANDB_ARTIFACTS:\n", + " try:\n", + " import wandb\n", + "\n", + " eval_run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, job_type=\"evaluation\", reinit=True)\n", + " eval_run.log({f\"eval/{k}\": v for k, v in summary.items()})\n", + " eval_run.log(\n", + " {\n", + " \"eval/objective_histogram\": wandb.Image(str(hist_path)),\n", + " \"eval/objective_scatter\": wandb.Image(str(scatter_path)),\n", + " \"eval/per_sample_table\": wandb.Table(dataframe=results),\n", + " }\n", + " )\n", + "\n", + " eval_artifact = wandb.Artifact(\"dcc26_beams2d_eval_report\", type=\"evaluation\")\n", + " for p in [results_path, summary_path, hist_path, scatter_path]:\n", + " eval_artifact.add_file(str(p))\n", + " eval_run.log_artifact(eval_artifact, aliases=[WANDB_ARTIFACT_ALIAS])\n", + " eval_run.finish()\n", + " print(\"Logged evaluation outputs to W&B.\")\n", + " except Exception as exc:\n", + " print(\"W&B evaluation logging failed (continuing locally):\", exc)" + ] + }, + { + "cell_type": "markdown", + "id": "786a08c7", + "metadata": {}, + "source": [ + "### Step 5 - Visual comparison for interpretation\n", + "\n", + "Pairwise plots help explain where scalar metrics are insufficient.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "417a4ca9", + "metadata": {}, + "outputs": [], + "source": [ + "# Visual side-by-side sample grid\n", + "fig, axes = plt.subplots(3, 4, figsize=(12, 8))\n", + "for i, ax in enumerate(axes.ravel()):\n", + " if i >= 12:\n", + " break\n", + " pair_idx = i // 2\n", + " if i % 2 == 0:\n", + " ax.imshow(gen_designs[pair_idx], cmap=\"gray\", vmin=0, vmax=1)\n", + " ax.set_title(f\"gen {pair_idx}\")\n", + " else:\n", + " ax.imshow(baseline_designs[pair_idx], cmap=\"gray\", vmin=0, vmax=1)\n", + " ax.set_title(f\"base {pair_idx}\")\n", + " ax.axis(\"off\")\n", + "fig.tight_layout()\n", + "fig.savefig(grid_path, dpi=150)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3aca41c8", + "metadata": {}, + "source": [ + "## Interpretation hints and discussion prompts\n", + "\n", + "Interpretation rule: improvements are compelling only if feasibility remains acceptable.\n", + "Use novelty/diversity to discuss whether the method generalizes or imitates.\n" + ] + }, + { + "cell_type": "markdown", + "id": "649db8c8", + "metadata": {}, + "source": [ + "## Discussion bridge to workshop breakout\n", + "\n", + "Prepare one metric-driven insight and one benchmark-design critique for group discussion.\n" + ] + }, + { + "cell_type": "markdown", + "id": "3d7e01e3", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "0144c8ad", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/solutions/03_add_new_problem_scaffold.ipynb b/workshops/dcc26/solutions/03_add_new_problem_scaffold.ipynb new file mode 100644 index 0000000..50de285 --- /dev/null +++ b/workshops/dcc26/solutions/03_add_new_problem_scaffold.ipynb @@ -0,0 +1,930 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ab4a0f1", + "metadata": {}, + "source": [ + "# Notebook 03: Add a New Problem Scaffold (DCC26)\n", + "\n", + "Reference implementation of a minimal, reproducible benchmark problem scaffold.\n" + ] + }, + { + "cell_type": "markdown", + "id": "b3e4b78c", + "metadata": {}, + "source": [ + "**Edit-safe start:** this notebook opens from GitHub in read-only source mode. Use **File -> Save a copy in Drive** before running edits so your changes stay in your own workspace.\n" + ] + }, + { + "cell_type": "markdown", + "id": "e5791875", + "metadata": {}, + "source": [ + "## Notebook map\n", + "\n", + "This notebook is written as a standalone lab chapter:\n", + "- context first,\n", + "- implementation second,\n", + "- interpretation third.\n", + "\n", + "If you are following asynchronously, run cells in order and use the success checks to validate each stage before moving on.\n" + ] + }, + { + "cell_type": "markdown", + "id": "cef985ef", + "metadata": {}, + "source": [ + "## Standalone guide\n", + "\n", + "Use this as a pattern for structuring new benchmark problems with explicit contracts and validation checks.\n" + ] + }, + { + "cell_type": "markdown", + "id": "5e7ab8b8", + "metadata": {}, + "source": [ + "## What makes a new problem benchmark-ready\n", + "\n", + "Benchmark value comes from clarity and comparability, not only simulator sophistication.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d6c6ea1", + "metadata": {}, + "outputs": [], + "source": [ + "# Colab/local dependency bootstrap\n", + "import subprocess\n", + "import sys\n", + "\n", + "IN_COLAB = \"google.colab\" in sys.modules\n", + "FORCE_INSTALL = False # Set True to force install outside Colab\n", + "\n", + "\n", + "def pip_install(packages: list[str]):\n", + " cmd = [sys.executable, \"-m\", \"pip\", \"install\", *packages]\n", + " print(\"Running:\", \" \".join(cmd))\n", + " subprocess.check_call(cmd)\n", + "\n", + "\n", + "BASE_PACKAGES = [\"engibench[beams2d]\", \"matplotlib\", \"gymnasium\", \"pybullet\"]\n", + "ENGIOPT_GIT = \"git+https://github.com/IDEALLab/EngiOpt.git@codex/dcc26-workshop-notebooks#egg=engiopt\"\n", + "\n", + "if IN_COLAB or FORCE_INSTALL:\n", + " print(\"Installing dependencies...\")\n", + " pip_install(BASE_PACKAGES)\n", + " pip_install([ENGIOPT_GIT])\n", + "\n", + " try:\n", + " import torch # noqa: F401\n", + " except Exception:\n", + " pip_install([\"torch\", \"torchvision\"])\n", + "\n", + " print(\"Dependency install complete.\")\n", + "else:\n", + " print(\"Skipping install (using current environment). Set FORCE_INSTALL=True to install here.\")" + ] + }, + { + "cell_type": "markdown", + "id": "0bbd0ad4", + "metadata": {}, + "source": [ + "### Step 1 - Import scaffold dependencies\n", + "\n", + "Keep imports minimal and interface-focused.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f10f2643", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "from dataclasses import dataclass\n", + "from typing import Annotated\n", + "\n", + "import numpy as np\n", + "from gymnasium import spaces\n", + "\n", + "from engibench.constraint import bounded\n", + "from engibench.constraint import constraint\n", + "from engibench.core import ObjectiveDirection\n", + "from engibench.core import OptiStep\n", + "from engibench.core import Problem\n", + "\n", + "import pybullet as p" + ] + }, + { + "cell_type": "markdown", + "id": "ea867bdb", + "metadata": {}, + "source": [ + "### Step 2 - Implement PyBullet manipulator co-design problem contract\n", + "\n", + "Ensure methods are deterministic and constraints/objectives are semantically explicit.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "402d904d", + "metadata": {}, + "outputs": [], + "source": [ + "class PlanarManipulatorCoDesignProblem(Problem[np.ndarray]):\n", + " \"\"\"Robotics co-design scaffold using a real PyBullet rollout loop.\"\"\"\n", + "\n", + " version = 0\n", + " objectives = (\n", + " (\"final_tracking_error_m\", ObjectiveDirection.MINIMIZE),\n", + " (\"actuation_energy_j\", ObjectiveDirection.MINIMIZE),\n", + " )\n", + "\n", + " @dataclass\n", + " class Conditions:\n", + " target_x: Annotated[float, bounded(lower=0.20, upper=1.35)] = 0.85\n", + " target_y: Annotated[float, bounded(lower=0.05, upper=1.20)] = 0.45\n", + " payload_kg: Annotated[float, bounded(lower=0.0, upper=2.0)] = 0.8\n", + " disturbance_scale: Annotated[float, bounded(lower=0.0, upper=0.30)] = 0.05\n", + "\n", + " @dataclass\n", + " class Config(Conditions):\n", + " sim_steps: Annotated[int, bounded(lower=60, upper=1200)] = 240\n", + " dt: Annotated[float, bounded(lower=1e-4, upper=0.05)] = 1.0 / 120.0\n", + " torque_limit: Annotated[float, bounded(lower=1.0, upper=50.0)] = 12.0\n", + " max_iter: Annotated[int, bounded(lower=1, upper=300)] = 60\n", + "\n", + " dataset_id = \"IDEALLab/planar_manipulator_codesign_v0\" # placeholder for future dataset integration\n", + " container_id = None\n", + "\n", + " def __init__(self, seed: int = 0, **kwargs):\n", + " super().__init__(seed=seed)\n", + " self.config = self.Config(**kwargs)\n", + " self.conditions = self.Conditions(\n", + " target_x=self.config.target_x,\n", + " target_y=self.config.target_y,\n", + " payload_kg=self.config.payload_kg,\n", + " disturbance_scale=self.config.disturbance_scale,\n", + " )\n", + "\n", + " # Design vector = [link1_m, link2_m, motor_strength, kp, kd, damping]\n", + " self.design_space = spaces.Box(\n", + " low=np.array([0.25, 0.20, 2.0, 5.0, 0.2, 0.0], dtype=np.float32),\n", + " high=np.array([1.00, 0.95, 30.0, 120.0, 18.0, 1.5], dtype=np.float32),\n", + " dtype=np.float32,\n", + " )\n", + "\n", + " @constraint\n", + " def reachable_workspace(design: np.ndarray, target_x: float, target_y: float, **_) -> None:\n", + " l1, l2 = float(design[0]), float(design[1])\n", + " r = float(np.sqrt(target_x**2 + target_y**2))\n", + " assert l1 + l2 >= r + 0.03, f\"target radius {r:.3f} exceeds reach {l1 + l2:.3f}\"\n", + "\n", + " @constraint\n", + " def gain_consistency(design: np.ndarray, **_) -> None:\n", + " kp, kd = float(design[3]), float(design[4])\n", + " assert kd <= 2.2 * np.sqrt(max(kp, 1e-6)), f\"kd={kd:.3f} too high for kp={kp:.3f}\"\n", + "\n", + " self.design_constraints = [reachable_workspace, gain_consistency]\n", + "\n", + " def _build_robot(self, l1: float, l2: float, payload_kg: float, damping: float) -> tuple[int, int]:\n", + " p.resetSimulation()\n", + " p.setGravity(0, 0, -9.81)\n", + "\n", + " link_masses = [0.5 + 0.2 * payload_kg, 0.35 + 0.25 * payload_kg]\n", + " link_collision = [-1, -1]\n", + " link_visual = [\n", + " p.createVisualShape(p.GEOM_CAPSULE, radius=0.025, length=l1, rgbaColor=[0.2, 0.5, 0.9, 1.0]),\n", + " p.createVisualShape(p.GEOM_CAPSULE, radius=0.020, length=l2, rgbaColor=[0.9, 0.4, 0.2, 1.0]),\n", + " ]\n", + " qx = p.getQuaternionFromEuler([0.0, np.pi / 2.0, 0.0])\n", + "\n", + " robot = p.createMultiBody(\n", + " baseMass=0.0,\n", + " baseCollisionShapeIndex=-1,\n", + " baseVisualShapeIndex=-1,\n", + " basePosition=[0, 0, 0],\n", + " linkMasses=link_masses,\n", + " linkCollisionShapeIndices=link_collision,\n", + " linkVisualShapeIndices=link_visual,\n", + " linkPositions=[[0, 0, 0], [l1, 0, 0]],\n", + " linkOrientations=[qx, qx],\n", + " linkInertialFramePositions=[[l1 / 2.0, 0, 0], [l2 / 2.0, 0, 0]],\n", + " linkInertialFrameOrientations=[[0, 0, 0, 1], [0, 0, 0, 1]],\n", + " linkParentIndices=[0, 1],\n", + " linkJointTypes=[p.JOINT_REVOLUTE, p.JOINT_REVOLUTE],\n", + " linkJointAxis=[[0, 0, 1], [0, 0, 1]],\n", + " )\n", + "\n", + " for j in [0, 1]:\n", + " p.changeDynamics(robot, j, linearDamping=0.0, angularDamping=float(damping))\n", + "\n", + " return robot, 1\n", + "\n", + " def _inverse_kinematics_2link(self, x: float, y: float, l1: float, l2: float) -> tuple[float, float]:\n", + " r2 = x * x + y * y\n", + " c2 = (r2 - l1 * l1 - l2 * l2) / (2.0 * l1 * l2)\n", + " c2 = float(np.clip(c2, -1.0, 1.0))\n", + " s2 = float(np.sqrt(max(0.0, 1.0 - c2 * c2)))\n", + " q2 = float(np.arctan2(s2, c2))\n", + " q1 = float(np.arctan2(y, x) - np.arctan2(l2 * s2, l1 + l2 * c2))\n", + " return q1, q2\n", + "\n", + " def _forward_kinematics_2link(self, q1: float, q2: float, l1: float, l2: float) -> tuple[float, float]:\n", + " x = l1 * np.cos(q1) + l2 * np.cos(q1 + q2)\n", + " y = l1 * np.sin(q1) + l2 * np.sin(q1 + q2)\n", + " return float(x), float(y)\n", + "\n", + " def _rollout(self, design: np.ndarray, cfg: dict, return_trace: bool = False):\n", + " l1, l2, motor_strength, kp, kd, damping = [float(v) for v in design]\n", + "\n", + " cid = p.connect(p.DIRECT)\n", + " try:\n", + " robot, _ = self._build_robot(l1, l2, cfg[\"payload_kg\"], damping)\n", + " q1_t, q2_t = self._inverse_kinematics_2link(cfg[\"target_x\"], cfg[\"target_y\"], l1, l2)\n", + "\n", + " err_trace = []\n", + " tau_trace = []\n", + " ee_trace = []\n", + " energy = 0.0\n", + "\n", + " for _step in range(int(cfg[\"sim_steps\"])):\n", + " for j, q_t in enumerate([q1_t, q2_t]):\n", + " p.setJointMotorControl2(\n", + " bodyUniqueId=robot,\n", + " jointIndex=j,\n", + " controlMode=p.POSITION_CONTROL,\n", + " targetPosition=q_t,\n", + " positionGain=float(kp) / 120.0,\n", + " velocityGain=float(kd) / 50.0,\n", + " force=float(cfg[\"torque_limit\"]) * float(motor_strength),\n", + " )\n", + "\n", + " if cfg[\"disturbance_scale\"] > 0:\n", + " disturb = self.np_random.normal(0.0, cfg[\"disturbance_scale\"], size=2)\n", + " p.applyExternalTorque(robot, 0, [0, 0, float(disturb[0])], p.LINK_FRAME)\n", + " p.applyExternalTorque(robot, 1, [0, 0, float(disturb[1])], p.LINK_FRAME)\n", + "\n", + " p.stepSimulation()\n", + "\n", + " js0 = p.getJointState(robot, 0)\n", + " js1 = p.getJointState(robot, 1)\n", + " q1, q2 = float(js0[0]), float(js1[0])\n", + " dq1, dq2 = float(js0[1]), float(js1[1])\n", + " tau1, tau2 = float(js0[3]), float(js1[3])\n", + "\n", + " ee_x, ee_y = self._forward_kinematics_2link(q1, q2, l1, l2)\n", + " err = float(np.sqrt((ee_x - cfg[\"target_x\"]) ** 2 + (ee_y - cfg[\"target_y\"]) ** 2))\n", + "\n", + " err_trace.append(err)\n", + " tau_trace.append((tau1, tau2))\n", + " ee_trace.append((ee_x, ee_y))\n", + " energy += (abs(tau1 * dq1) + abs(tau2 * dq2)) * float(cfg[\"dt\"])\n", + "\n", + " final_error = float(err_trace[-1])\n", + " obj = np.array([final_error, float(energy)], dtype=np.float32)\n", + "\n", + " if return_trace:\n", + " trace = {\n", + " \"ee_trace\": np.array(ee_trace, dtype=np.float32),\n", + " \"err_trace\": np.array(err_trace, dtype=np.float32),\n", + " \"tau_trace\": np.array(tau_trace, dtype=np.float32),\n", + " \"target\": np.array([cfg[\"target_x\"], cfg[\"target_y\"]], dtype=np.float32),\n", + " \"design\": np.array(design, dtype=np.float32),\n", + " \"objectives\": obj,\n", + " }\n", + " return obj, trace\n", + "\n", + " return obj\n", + " finally:\n", + " p.disconnect(cid)\n", + "\n", + " def simulate(self, design: np.ndarray, config: dict | None = None) -> np.ndarray:\n", + " cfg = {**self.config.__dict__, **(config or {})}\n", + " x = np.clip(design.astype(np.float32), self.design_space.low, self.design_space.high)\n", + " return self._rollout(x, cfg, return_trace=False)\n", + "\n", + " def optimize(self, starting_point: np.ndarray, config: dict | None = None):\n", + " cfg = {**self.config.__dict__, **(config or {})}\n", + " x = np.clip(starting_point.astype(np.float32), self.design_space.low, self.design_space.high)\n", + "\n", + " best = x.copy()\n", + " best_obj = self.simulate(best, cfg)\n", + " best_score = float(best_obj[0] + 0.02 * best_obj[1])\n", + "\n", + " history = [OptiStep(obj_values=best_obj, step=0)]\n", + " step_scale = np.array([0.05, 0.05, 2.5, 8.0, 1.2, 0.08], dtype=np.float32)\n", + "\n", + " for step in range(1, int(cfg[\"max_iter\"]) + 1):\n", + " candidate = best + self.np_random.normal(0.0, 1.0, size=6).astype(np.float32) * step_scale\n", + " candidate = np.clip(candidate, self.design_space.low, self.design_space.high)\n", + "\n", + " if self.check_constraints(candidate, cfg):\n", + " history.append(OptiStep(obj_values=np.array([np.inf, np.inf], dtype=np.float32), step=step))\n", + " continue\n", + "\n", + " obj = self.simulate(candidate, cfg)\n", + " score = float(obj[0] + 0.02 * obj[1])\n", + " if score < best_score:\n", + " best, best_obj, best_score = candidate, obj, score\n", + "\n", + " history.append(OptiStep(obj_values=best_obj, step=step))\n", + "\n", + " return best, history\n", + "\n", + " def render(self, design: np.ndarray, *, open_window: bool = False):\n", + " import matplotlib.pyplot as plt\n", + "\n", + " cfg = self.config.__dict__\n", + " x = np.clip(design.astype(np.float32), self.design_space.low, self.design_space.high)\n", + " obj, trace = self._rollout(x, cfg, return_trace=True)\n", + "\n", + " ee = trace[\"ee_trace\"]\n", + " err = trace[\"err_trace\"]\n", + " target = trace[\"target\"]\n", + " tau = trace[\"tau_trace\"]\n", + "\n", + " fig, axes = plt.subplots(1, 4, figsize=(17, 4.2))\n", + "\n", + " labels = [\"link1\", \"link2\", \"motor\", \"kp\", \"kd\", \"damping\"]\n", + " axes[0].bar(labels, x, color=[\"#4c78a8\", \"#4c78a8\", \"#f58518\", \"#54a24b\", \"#e45756\", \"#72b7b2\"])\n", + " axes[0].set_title(\"Design variables\")\n", + " axes[0].tick_params(axis=\"x\", rotation=35)\n", + "\n", + " axes[1].plot(ee[:, 0], ee[:, 1], lw=2, label=\"end-effector path\")\n", + " axes[1].scatter([target[0]], [target[1]], c=\"red\", marker=\"x\", s=70, label=\"target\")\n", + " r = x[0] + x[1]\n", + " circle = plt.Circle((0, 0), r, color=\"gray\", fill=False, linestyle=\"--\", alpha=0.5)\n", + " axes[1].add_patch(circle)\n", + " axes[1].set_aspect(\"equal\", \"box\")\n", + " axes[1].set_title(\"Task-space trajectory\")\n", + " axes[1].set_xlabel(\"x [m]\")\n", + " axes[1].set_ylabel(\"y [m]\")\n", + " axes[1].legend(fontsize=8)\n", + "\n", + " axes[2].plot(err, color=\"#e45756\")\n", + " axes[2].set_title(\"Tracking error over time\")\n", + " axes[2].set_xlabel(\"step\")\n", + " axes[2].set_ylabel(\"error [m]\")\n", + " axes[2].grid(alpha=0.3)\n", + "\n", + " axes[3].plot(np.abs(tau[:, 0]), label=\"|tau1|\")\n", + " axes[3].plot(np.abs(tau[:, 1]), label=\"|tau2|\")\n", + " axes[3].set_title(\"Actuation effort\")\n", + " axes[3].set_xlabel(\"step\")\n", + " axes[3].set_ylabel(\"torque [Nm]\")\n", + " axes[3].legend(fontsize=8)\n", + " axes[3].grid(alpha=0.3)\n", + "\n", + " fig.suptitle(\n", + " f\"Objectives: final_error={obj[0]:.4f} m, energy={obj[1]:.3f} J\",\n", + " y=1.03,\n", + " )\n", + " fig.tight_layout()\n", + "\n", + " if open_window:\n", + " plt.show()\n", + " return fig, axes\n", + "\n", + " def random_design(self):\n", + " d = self.np_random.uniform(self.design_space.low, self.design_space.high).astype(np.float32)\n", + " return d, -1" + ] + }, + { + "cell_type": "markdown", + "id": "2b9572ae", + "metadata": {}, + "source": [ + "### Step 3 - Smoke-test the scaffold\n", + "\n", + "Validate behavior with simple checks before scaling to real domains.\n", + "\n", + "\n", + "Use the multi-panel render to read **where heat enters**, **how material is distributed**, and **where thermal bottlenecks remain**.\n", + "\n", + "\n", + "Use the final figure to interpret whether the design/controller combination reaches the target robustly with acceptable energy use.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72a40876", + "metadata": {}, + "outputs": [], + "source": [ + "problem = PlanarManipulatorCoDesignProblem(\n", + " seed=42,\n", + " target_x=0.9,\n", + " target_y=0.45,\n", + " payload_kg=0.8,\n", + " disturbance_scale=0.04,\n", + " sim_steps=220,\n", + " max_iter=40,\n", + ")\n", + "start, _ = problem.random_design()\n", + "\n", + "cfg = {\n", + " \"target_x\": 0.9,\n", + " \"target_y\": 0.45,\n", + " \"payload_kg\": 0.8,\n", + " \"disturbance_scale\": 0.04,\n", + " \"sim_steps\": 220,\n", + " \"dt\": 1.0 / 120.0,\n", + " \"torque_limit\": 12.0,\n", + " \"max_iter\": 40,\n", + "}\n", + "\n", + "print(\"design space:\", problem.design_space)\n", + "print(\"objectives:\", problem.objectives)\n", + "print(\"conditions:\", problem.conditions)\n", + "\n", + "viol = problem.check_constraints(start, config=cfg)\n", + "print(\"constraint violations:\", len(viol))\n", + "\n", + "obj0 = problem.simulate(start, config=cfg)\n", + "opt_design, history = problem.optimize(start, config=cfg)\n", + "objf = problem.simulate(opt_design, config=cfg)\n", + "\n", + "print(\"initial objectives [tracking_error_m, energy_J]:\", obj0.tolist())\n", + "print(\"final objectives [tracking_error_m, energy_J]:\", objf.tolist())\n", + "print(\"optimization steps:\", len(history))\n", + "print(\"How to read plots: vars | task-space path | error timeline | torque timeline\")\n", + "\n", + "problem.render(opt_design)" + ] + }, + { + "cell_type": "markdown", + "id": "9ec93ba4", + "metadata": {}, + "source": [ + "## Mapping to real EngiBench contributions\n", + "\n", + "Use this template to onboard new domains while preserving common evaluation semantics.\n" + ] + }, + { + "cell_type": "markdown", + "id": "2e15e82b", + "metadata": {}, + "source": [ + "## Contribution checklist\n", + "\n", + "Check for leakage risks, undocumented defaults, and missing reproducibility metadata before contribution.\n" + ] + }, + { + "cell_type": "markdown", + "id": "750f0558", + "metadata": {}, + "source": [ + "## Troubleshooting\n", + "\n", + "If a section fails, do not continue downstream. Fix locally first, then rerun the section and its immediate checks.\n", + "This notebook is intentionally staged so failures are localized.\n" + ] + }, + { + "cell_type": "markdown", + "id": "afcfd2f2", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "Before closing, record three points:\n", + "1. What conclusion is directly supported by your metrics?\n", + "2. What remains uncertain (and why)?\n", + "3. What extra experiment would you run next to reduce that uncertainty?\n" + ] + }, + { + "cell_type": "markdown", + "id": "206a966d", + "metadata": {}, + "source": [ + "## Optional extension - Build dataset and train an EngiOpt generative model\n", + "\n", + "This extension runs a full offline loop using an **existing EngiOpt model** (`cgan_1d`) on top of your custom simulator problem:\n", + "\n", + "1. Generate a feasible dataset from simulator rollouts.\n", + "2. Keep a top-performing subset.\n", + "3. Train EngiOpt `cgan_1d` (`Generator` + `Discriminator`) for conditional generation.\n", + "4. Compare generated designs vs a random-design baseline.\n", + "\n", + "Why this is useful:\n", + "- It demonstrates reuse of existing model infrastructure from EngiOpt.\n", + "- It clarifies how to adapt EngiOpt model classes to new, custom problem scaffolds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9054cfce", + "metadata": {}, + "outputs": [], + "source": [ + "# Optional extension controls (safe defaults)\n", + "from pathlib import Path\n", + "import sys\n", + "\n", + "import numpy as np\n", + "import torch as th\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "RUN_OPTIONAL_SECTION = False # Set True to run this optional extension\n", + "N_FEASIBLE_SAMPLES = 260\n", + "TOP_FRACTION = 0.35\n", + "EPOCHS = 30\n", + "BATCH_SIZE = 64\n", + "LATENT_DIM = 8\n", + "FAST_SIM_CFG = {\"sim_steps\": 80, \"dt\": 1.0 / 120.0}\n", + "EVAL_SAMPLES = 40\n", + "\n", + "if \"problem\" not in globals():\n", + " problem = PlanarManipulatorCoDesignProblem(seed=7)\n", + "\n", + "if \"google.colab\" in sys.modules:\n", + " OPTIONAL_ARTIFACT_DIR = Path(\"/content/dcc26_optional_artifacts\")\n", + "else:\n", + " OPTIONAL_ARTIFACT_DIR = Path(\"workshops/dcc26/optional_artifacts\")\n", + "OPTIONAL_ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(f\"Optional artifacts dir: {OPTIONAL_ARTIFACT_DIR.resolve()}\")\n", + "print(\"Optional section enabled:\" if RUN_OPTIONAL_SECTION else \"Optional section disabled:\", RUN_OPTIONAL_SECTION)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9780c307", + "metadata": {}, + "outputs": [], + "source": [ + "# Build offline dataset from simulator rollouts\n", + "rng = np.random.default_rng(123)\n", + "\n", + "\n", + "def sample_condition_dict() -> dict:\n", + " return {\n", + " \"target_x\": float(rng.uniform(0.20, 1.35)),\n", + " \"target_y\": float(rng.uniform(0.05, 1.20)),\n", + " \"payload_kg\": float(rng.uniform(0.0, 2.0)),\n", + " \"disturbance_scale\": float(rng.uniform(0.0, 0.30)),\n", + " }\n", + "\n", + "\n", + "def cond_to_vec(cfg: dict) -> np.ndarray:\n", + " return np.array(\n", + " [\n", + " cfg[\"target_x\"],\n", + " cfg[\"target_y\"],\n", + " cfg[\"payload_kg\"],\n", + " cfg[\"disturbance_scale\"],\n", + " ],\n", + " dtype=np.float32,\n", + " )\n", + "\n", + "\n", + "def objective_score(obj: np.ndarray) -> float:\n", + " return float(obj[0] + 0.02 * obj[1])\n", + "\n", + "\n", + "def make_dataset(problem_obj, n_feasible: int):\n", + " designs, conds, objs = [], [], []\n", + " max_attempts = n_feasible * 8\n", + " attempts = 0\n", + "\n", + " while len(designs) < n_feasible and attempts < max_attempts:\n", + " attempts += 1\n", + " d, _ = problem_obj.random_design()\n", + " cfg = sample_condition_dict()\n", + "\n", + " if len(problem_obj.check_constraints(d, cfg)) > 0:\n", + " continue\n", + "\n", + " obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " designs.append(d.astype(np.float32))\n", + " conds.append(cond_to_vec(cfg))\n", + " objs.append(obj.astype(np.float32))\n", + "\n", + " if len(designs) % 40 == 0:\n", + " print(f\"Collected feasible samples: {len(designs)}/{n_feasible}\")\n", + "\n", + " if len(designs) < max(48, n_feasible // 3):\n", + " raise RuntimeError(f\"Not enough feasible samples ({len(designs)}).\")\n", + "\n", + " designs = np.stack(designs)\n", + " conds = np.stack(conds)\n", + " objs = np.stack(objs)\n", + " scores = np.array([objective_score(o) for o in objs], dtype=np.float32)\n", + "\n", + " keep_n = max(48, int(TOP_FRACTION * len(scores)))\n", + " top_idx = np.argsort(scores)[:keep_n]\n", + "\n", + " data = {\n", + " \"designs_all\": designs,\n", + " \"conditions_all\": conds,\n", + " \"objectives_all\": objs,\n", + " \"scores_all\": scores,\n", + " \"designs_top\": designs[top_idx],\n", + " \"conditions_top\": conds[top_idx],\n", + " \"objectives_top\": objs[top_idx],\n", + " \"scores_top\": scores[top_idx],\n", + " }\n", + " return data\n", + "\n", + "\n", + "if RUN_OPTIONAL_SECTION:\n", + " dataset = make_dataset(problem, N_FEASIBLE_SAMPLES)\n", + " np.savez(OPTIONAL_ARTIFACT_DIR / \"manipulator_dataset.npz\", **dataset)\n", + " print(\"Saved dataset:\", OPTIONAL_ARTIFACT_DIR / \"manipulator_dataset.npz\")\n", + " print(\"All samples:\", dataset[\"designs_all\"].shape[0], \"| Top samples:\", dataset[\"designs_top\"].shape[0])\n", + "else:\n", + " dataset = None\n", + " print(\"Skipped dataset creation. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "markdown", + "id": "57dac117", + "metadata": {}, + "source": [ + "### Optional model - EngiOpt `cgan_1d`\n", + "\n", + "This cell reuses `engiopt.cgan_1d.cgan_1d` classes directly:\n", + "- `Normalizer`\n", + "- `Generator`\n", + "- `Discriminator`\n", + "\n", + "Adapter note:\n", + "- `Discriminator` in this module expects module-level `design_shape` and `n_conds` symbols.\n", + "- We set those explicitly before instantiation to keep behavior aligned with the original script.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1d6074e", + "metadata": {}, + "outputs": [], + "source": [ + "# Train EngiOpt cgan_1d on the top-performing subset\n", + "if RUN_OPTIONAL_SECTION:\n", + " import engiopt.cgan_1d.cgan_1d as cgan1d\n", + "\n", + " device = th.device(\"cuda\" if th.cuda.is_available() else \"cpu\")\n", + "\n", + " x_cond = dataset[\"conditions_top\"].astype(np.float32)\n", + " y_design = dataset[\"designs_top\"].astype(np.float32)\n", + "\n", + " cond_t = th.tensor(x_cond, dtype=th.float32, device=device)\n", + " design_t = th.tensor(y_design, dtype=th.float32, device=device)\n", + "\n", + " cond_min = cond_t.amin(dim=0)\n", + " cond_max = cond_t.amax(dim=0)\n", + " design_min = design_t.amin(dim=0)\n", + " design_max = design_t.amax(dim=0)\n", + "\n", + " conds_normalizer = cgan1d.Normalizer(cond_min, cond_max)\n", + " design_normalizer = cgan1d.Normalizer(design_min, design_max)\n", + "\n", + " design_shape = (design_t.shape[1],)\n", + " n_conds = cond_t.shape[1]\n", + "\n", + " # Compatibility shim for cgan_1d.Discriminator internal references.\n", + " cgan1d.design_shape = design_shape\n", + " cgan1d.n_conds = n_conds\n", + "\n", + " generator = cgan1d.Generator(\n", + " latent_dim=LATENT_DIM,\n", + " n_conds=n_conds,\n", + " design_shape=design_shape,\n", + " design_normalizer=design_normalizer,\n", + " conds_normalizer=conds_normalizer,\n", + " ).to(device)\n", + "\n", + " discriminator = cgan1d.Discriminator(\n", + " conds_normalizer=conds_normalizer,\n", + " design_normalizer=design_normalizer,\n", + " ).to(device)\n", + "\n", + " loader = DataLoader(TensorDataset(design_t, cond_t), batch_size=BATCH_SIZE, shuffle=True)\n", + "\n", + " adv_loss = nn.BCELoss()\n", + " opt_g = th.optim.Adam(generator.parameters(), lr=2e-4, betas=(0.5, 0.999))\n", + " opt_d = th.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.5, 0.999))\n", + "\n", + " g_hist, d_hist = [], []\n", + "\n", + " for epoch in range(1, EPOCHS + 1):\n", + " g_epoch, d_epoch, n_steps = 0.0, 0.0, 0\n", + " for real_design, cond in loader:\n", + " bs = real_design.shape[0]\n", + " valid = th.ones((bs, 1), device=device)\n", + " fake = th.zeros((bs, 1), device=device)\n", + "\n", + " # Generator update\n", + " opt_g.zero_grad()\n", + " z = th.randn((bs, LATENT_DIM), device=device)\n", + " gen_design = generator(z, cond)\n", + " g_loss = adv_loss(discriminator(gen_design, cond), valid)\n", + " g_loss.backward()\n", + " opt_g.step()\n", + "\n", + " # Discriminator update\n", + " opt_d.zero_grad()\n", + " real_loss = adv_loss(discriminator(real_design, cond), valid)\n", + " fake_loss = adv_loss(discriminator(gen_design.detach(), cond), fake)\n", + " d_loss = 0.5 * (real_loss + fake_loss)\n", + " d_loss.backward()\n", + " opt_d.step()\n", + "\n", + " g_epoch += float(g_loss.item())\n", + " d_epoch += float(d_loss.item())\n", + " n_steps += 1\n", + "\n", + " g_hist.append(g_epoch / max(1, n_steps))\n", + " d_hist.append(d_epoch / max(1, n_steps))\n", + " if epoch == 1 or epoch % 5 == 0 or epoch == EPOCHS:\n", + " print(f\"Epoch {epoch:02d}/{EPOCHS} | g_loss={g_hist[-1]:.6f} | d_loss={d_hist[-1]:.6f}\")\n", + "\n", + " th.save(\n", + " {\n", + " \"generator\": generator.state_dict(),\n", + " \"discriminator\": discriminator.state_dict(),\n", + " \"cond_min\": cond_min.cpu(),\n", + " \"cond_max\": cond_max.cpu(),\n", + " \"design_min\": design_min.cpu(),\n", + " \"design_max\": design_max.cpu(),\n", + " },\n", + " OPTIONAL_ARTIFACT_DIR / \"engiopt_cgan1d_weights.pt\",\n", + " )\n", + " print(\"Saved model:\", OPTIONAL_ARTIFACT_DIR / \"engiopt_cgan1d_weights.pt\")\n", + "else:\n", + " generator, discriminator = None, None\n", + " g_hist, d_hist = [], []\n", + " device = th.device(\"cpu\")\n", + " print(\"Skipped model training. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5469eb4", + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate generated designs vs random baseline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "def sample_baseline(problem_obj, cfg: dict, trials: int = 8):\n", + " best_obj = None\n", + " for _ in range(trials):\n", + " d, _ = problem_obj.random_design()\n", + " if len(problem_obj.check_constraints(d, cfg)) > 0:\n", + " continue\n", + " obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " if best_obj is None or objective_score(obj) < objective_score(best_obj):\n", + " best_obj = obj\n", + " if best_obj is None:\n", + " d, _ = problem_obj.random_design()\n", + " best_obj = problem_obj.simulate(d, {**cfg, **FAST_SIM_CFG})\n", + " return best_obj\n", + "\n", + "\n", + "def generate_design(generator_obj, cfg_vec: np.ndarray, lb: np.ndarray, ub: np.ndarray):\n", + " # cgan_1d Generator uses BatchNorm; switch to eval for single-sample inference.\n", + " was_training = generator_obj.training\n", + " generator_obj.eval()\n", + " try:\n", + " with th.no_grad():\n", + " c = th.tensor(cfg_vec[None, :], dtype=th.float32, device=device)\n", + " z = th.randn((1, LATENT_DIM), dtype=th.float32, device=device)\n", + " d = generator_obj(z, c).cpu().numpy()[0]\n", + " finally:\n", + " if was_training:\n", + " generator_obj.train()\n", + " return np.clip(d.astype(np.float32), lb, ub)\n", + "\n", + "\n", + "if RUN_OPTIONAL_SECTION:\n", + " lb = problem.design_space.low.astype(np.float32)\n", + " ub = problem.design_space.high.astype(np.float32)\n", + "\n", + " gen_objs = []\n", + " base_objs = []\n", + " feasible_count = 0\n", + "\n", + " for _ in range(EVAL_SAMPLES):\n", + " cfg = sample_condition_dict()\n", + " cfg_vec = cond_to_vec(cfg)\n", + "\n", + " gen_obj = None\n", + " for _retry in range(6):\n", + " d_gen = generate_design(generator, cfg_vec, lb, ub)\n", + " if len(problem.check_constraints(d_gen, cfg)) == 0:\n", + " gen_obj = problem.simulate(d_gen, {**cfg, **FAST_SIM_CFG})\n", + " feasible_count += 1\n", + " break\n", + " if gen_obj is None:\n", + " d_fallback, _ = problem.random_design()\n", + " gen_obj = problem.simulate(d_fallback, {**cfg, **FAST_SIM_CFG})\n", + "\n", + " base_obj = sample_baseline(problem, cfg, trials=8)\n", + " gen_objs.append(gen_obj)\n", + " base_objs.append(base_obj)\n", + "\n", + " gen_objs = np.stack(gen_objs)\n", + " base_objs = np.stack(base_objs)\n", + "\n", + " summary = {\n", + " \"generated_error_mean\": float(np.mean(gen_objs[:, 0])),\n", + " \"generated_energy_mean\": float(np.mean(gen_objs[:, 1])),\n", + " \"baseline_error_mean\": float(np.mean(base_objs[:, 0])),\n", + " \"baseline_energy_mean\": float(np.mean(base_objs[:, 1])),\n", + " \"generated_feasible_rate\": float(feasible_count / EVAL_SAMPLES),\n", + " }\n", + "\n", + " print(\"Optional extension summary (EngiOpt cgan_1d):\")\n", + " for k, v in summary.items():\n", + " print(f\" {k}: {v:.6f}\")\n", + "\n", + " np.savez(\n", + " OPTIONAL_ARTIFACT_DIR / \"optional_eval_summary_engiopt_cgan1d.npz\",\n", + " gen_objs=gen_objs,\n", + " base_objs=base_objs,\n", + " g_hist=np.array(g_hist, dtype=np.float32),\n", + " d_hist=np.array(d_hist, dtype=np.float32),\n", + " **summary,\n", + " )\n", + "\n", + " fig, axes = plt.subplots(1, 3, figsize=(14, 4))\n", + "\n", + " axes[0].plot(g_hist, label=\"g_loss\")\n", + " axes[0].plot(d_hist, label=\"d_loss\")\n", + " axes[0].set_title(\"EngiOpt cgan_1d training losses\")\n", + " axes[0].set_xlabel(\"epoch\")\n", + " axes[0].legend()\n", + " axes[0].grid(alpha=0.3)\n", + "\n", + " axes[1].hist(base_objs[:, 0], bins=12, alpha=0.6, label=\"baseline\")\n", + " axes[1].hist(gen_objs[:, 0], bins=12, alpha=0.6, label=\"generated\")\n", + " axes[1].set_title(\"Final tracking error\")\n", + " axes[1].set_xlabel(\"error [m]\")\n", + " axes[1].legend()\n", + "\n", + " axes[2].hist(base_objs[:, 1], bins=12, alpha=0.6, label=\"baseline\")\n", + " axes[2].hist(gen_objs[:, 1], bins=12, alpha=0.6, label=\"generated\")\n", + " axes[2].set_title(\"Actuation energy\")\n", + " axes[2].set_xlabel(\"energy [J]\")\n", + " axes[2].legend()\n", + "\n", + " fig.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"Skipped evaluation. Set RUN_OPTIONAL_SECTION=True to run this block.\")" + ] + }, + { + "cell_type": "markdown", + "id": "90f7d4ff", + "metadata": {}, + "source": [ + "### Discussion prompts for workshop synthesis\n", + "\n", + "1. How portable are EngiOpt models across domains with different design representations?\n", + "2. What is the minimum adapter contract needed to reuse a model on a new problem?\n", + "3. Should benchmark reporting require both feasibility and objective trade-off metrics for generated designs?\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/workshops/dcc26/utils/__init__.py b/workshops/dcc26/utils/__init__.py new file mode 100644 index 0000000..bb5dd63 --- /dev/null +++ b/workshops/dcc26/utils/__init__.py @@ -0,0 +1 @@ +"""Utilities for DCC26 workshop notebooks.""" diff --git a/workshops/dcc26/utils/notebook_helpers.py b/workshops/dcc26/utils/notebook_helpers.py new file mode 100644 index 0000000..c1df904 --- /dev/null +++ b/workshops/dcc26/utils/notebook_helpers.py @@ -0,0 +1,49 @@ +"""Helper utilities for DCC26 workshop notebooks.""" + +from __future__ import annotations + +import json +import os +import random +from typing import Any + +import numpy as np +import torch as th + + +def set_global_seed(seed: int) -> None: + """Set seeds for reproducibility across numpy, python, and torch.""" + random.seed(seed) + np.random.seed(seed) + th.manual_seed(seed) + if th.cuda.is_available(): + th.cuda.manual_seed_all(seed) + th.backends.cudnn.deterministic = True + th.backends.cudnn.benchmark = False + + +def pick_device() -> th.device: + """Pick an available torch device in priority order.""" + if th.backends.mps.is_available(): + return th.device("mps") + if th.cuda.is_available(): + return th.device("cuda") + return th.device("cpu") + + +def ensure_dir(path: str) -> str: + """Ensure a directory exists and return the path.""" + os.makedirs(path, exist_ok=True) + return path + + +def save_json(data: Any, path: str) -> None: + """Save Python data as a JSON file.""" + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + + +def load_json(path: str) -> Any: + """Load Python data from a JSON file.""" + with open(path, encoding="utf-8") as f: + return json.load(f)