diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 3f7d15277..cc0814e00 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -53,6 +53,11 @@ jobs: - name: Render Quarto site run: quarto render + - name: Generate Jupyter notebooks and add to HTML + run: julia --project=assets/scripts/notebooks assets/scripts/notebooks/notebooks.jl + env: + PATH_PREFIX: /pr-previews/${{ github.event.pull_request.number }} + - name: Save _freeze folder id: cache-save if: ${{ !cancelled() }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5cb81b4f0..1b919415f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -85,6 +85,11 @@ jobs: - name: Render Quarto site run: quarto render + - name: Generate Jupyter notebooks and add to HTML + run: julia --project=assets/scripts/notebooks assets/scripts/notebooks/notebooks.jl + env: + PATH_PREFIX: /versions/${{ env.version }} + - name: Rename original search index run: mv _site/search.json _site/search_original.json diff --git a/.gitignore b/.gitignore index eef389952..f18a3be59 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ site_libs .DS_Store index_files digest.txt -*.bak \ No newline at end of file +**/*.quarto_ipynb +*.bak diff --git a/_quarto.yml b/_quarto.yml index 928dab58d..473fad7b1 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -117,7 +117,6 @@ website: - developers/compiler/model-manual/index.qmd - developers/compiler/minituring-compiler/index.qmd - developers/compiler/minituring-contexts/index.qmd - - developers/compiler/design-overview/index.qmd - section: "DynamicPPL Models" collapse-level: 1 @@ -224,7 +223,6 @@ contributing-guide: developers/contributing dev-model-manual: developers/compiler/model-manual contexts: developers/compiler/minituring-contexts minituring: developers/compiler/minituring-compiler -using-turing-compiler: developers/compiler/design-overview dev-variational-inference: developers/inference/variational-inference using-turing-implementing-samplers: developers/inference/implementing-samplers dev-transforms-distributions: developers/transforms/distributions diff --git a/assets/scripts/notebooks/.gitignore b/assets/scripts/notebooks/.gitignore new file mode 100644 index 000000000..ba39cc531 --- /dev/null +++ b/assets/scripts/notebooks/.gitignore @@ -0,0 +1 @@ +Manifest.toml diff --git a/assets/scripts/notebooks/Project.toml b/assets/scripts/notebooks/Project.toml new file mode 100644 index 000000000..f24afeabe --- /dev/null +++ b/assets/scripts/notebooks/Project.toml @@ -0,0 +1,3 @@ +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" diff --git a/assets/scripts/notebooks/notebooks.jl b/assets/scripts/notebooks/notebooks.jl new file mode 100644 index 000000000..24f635b3d --- /dev/null +++ b/assets/scripts/notebooks/notebooks.jl @@ -0,0 +1,258 @@ +using Pkg +Pkg.instantiate() + +using JSON +import JuliaSyntax + +abstract type Cell end +struct JuliaCodeCell <: Cell + code::String +end +function JSON.lower(cell::JuliaCodeCell) + return Dict( + "cell_type" => "code", + "source" => cell.code, + "metadata" => Dict(), + "outputs" => Any[], + "execution_count" => nothing, + ) +end +struct MarkdownCell <: Cell + content::String +end +function JSON.lower(cell::MarkdownCell) + return Dict( + "cell_type" => "markdown", + "source" => cell.content, + "metadata" => Dict(), + ) +end + +struct Notebook + cells::Vector{Cell} +end +function JSON.lower(nb::Notebook) + return Dict( + "cells" => [JSON.lower(cell) for cell in nb.cells], + "metadata" => Dict( + "kernelspec" => Dict( + "display_name" => "Julia", + "language" => "julia", + "name" => "julia" + ), + "language_info" => Dict( + "file_extension" => ".jl", + "mimetype" => "application/julia", + "name" => "julia" + ) + ), + "nbformat" => 4, + "nbformat_minor" => 5 + ) +end + +""" + fix_callouts(md_content::AbstractString)::String + +Convert Quarto callouts in `md_content` to blockquotes. +""" +function fix_callouts(md_content::AbstractString)::String + # Quarto callouts look like, for example, `::: {.callout-note}` + # There isn't a good Jupyter equivalent, so we'll just use blockquotes. + # https://github.com/quarto-dev/quarto-cli/issues/1167 + callout_regex = r"^:::\s*\{\.callout-\w+\}.*$" + callout_end_regex = r"^:::\s*$" + new_lines = String[] + in_callout = false + for line in split(md_content, '\n') + if in_callout + if occursin(callout_end_regex, line) + in_callout = false + else + push!(new_lines, "> " * line) + end + else + if occursin(callout_regex, line) + in_callout = true + else + push!(new_lines, line) + end + end + end + return join(new_lines, '\n') +end + +""" + parse_cells(qmd_path::String)::Notebook + +Parse a .qmd file. Returns a vector of `Cell` objects representing the code and markdown +cells, as well as a set of imported packages found in Julia code cells. +""" +function parse_cells(qmd_path::String)::Notebook + content = read(qmd_path, String) + + # Remove YAML front matter. + yaml_front_matter_regex = r"^---\n(.*?)\n---\n"s + content = replace(content, yaml_front_matter_regex => "") + content = strip(content) + + packages = Set{Symbol}() + # Extract code blocks. + executable_content_regex = r"```\{(\w+)\}(.*?)```"s + # These are Markdown cells. + markdown_cell_contents = split(content, executable_content_regex; keepempty=true) + # These are code cells + code_cell_contents = collect(eachmatch(executable_content_regex, content)) + # Because we set `keepempty=true`, `splits` will always have one more element than `matches`. + # We can interleave them to reconstruct the document structure. + cells = Cell[] + for (i, md_content) in enumerate(markdown_cell_contents) + md_content = strip(md_content) + if !isempty(md_content) + push!(cells, MarkdownCell(fix_callouts(md_content))) + end + if i <= length(code_cell_contents) + match = code_cell_contents[i] + lang = match.captures[1] + code = strip(match.captures[2]) + if lang == "julia" + cell = JuliaCodeCell(code) + push!(cells, cell) + union!(packages, extract_imports(cell)) + else + # There are some code cells that are not Julia for example + # dot and mermaid. You can see what cells there are with + # git grep -E '```\{.+\}' | grep -v julia + # For these cells we'll just convert to Markdown. + push!(cells, MarkdownCell("```$lang\n$code\n```")) + end + end + end + + # Prepend a cell to install the necessary packages + imports_as_string = join(["\"" * string(pkg) * "\"" for pkg in packages], ", ") + new_cell = JuliaCodeCell("# Install necessary dependencies.\nusing Pkg\nPkg.activate(; temp=true)\nPkg.add([$imports_as_string])") + cells = [new_cell, cells...] + + # And we're done! + return Notebook(cells) +end + +""" + extract_imports(cell::JuliaCodeCell)::Set{Symbol} + +Extract all packages that are imported inside `cell`. +""" +function extract_imports(cell::JuliaCodeCell)::Set{Symbol} + toplevel_expr = JuliaSyntax.parseall(Expr, cell.code) + imports = Set{Symbol}() + for expr in toplevel_expr.args + if expr isa Expr && (expr.head == :using || expr.head == :import) + for arg in expr.args + if arg isa Expr && arg.head == :. + push!(imports, arg.args[1]) + elseif arg isa Expr && arg.head == :(:) + subarg = arg.args[1] + if subarg isa Expr && subarg.head == :. + push!(imports, subarg.args[1]) + end + elseif arg isa Expr && arg.head == :as + subarg = arg.args[1] + if subarg isa Expr && subarg.head == :. + push!(imports, subarg.args[1]) + elseif subarg isa Symbol + push!(imports, subarg) + end + end + end + end + end + return imports +end + +function convert_qmd_to_ipynb(in_qmd_path::String, out_ipynb_path::String) + @info "converting $in_qmd_path to $out_ipynb_path..." + notebook = parse_cells(in_qmd_path) + JSON.json(out_ipynb_path, notebook; pretty=true) + @info " - done." +end + +function add_ipynb_link_to_html(html_path::String, ipynb_path::String) + # this would look like "getting-started.ipynb" and is used when downloading a notebook + SUGGESTED_FILENAME = basename(dirname(ipynb_path)) * ".ipynb" + # The Colab URL needs to look like + # https://colab.research.google.com/github/TuringLang/docs/blob/gh-pages/path/to/notebook.ipynb + # Because ipynb_path has `_site/` prefix, we need to strip that off. + ipynb_path_no_site = replace(ipynb_path, r"^_site/" => "") + PATH_PREFIX = get(ENV, "PATH_PREFIX", "") + COLAB_URL = "https://colab.research.google.com/github/TuringLang/docs/blob/gh-pages$PATH_PREFIX/$ipynb_path_no_site" + @info "adding link to ipynb notebook in $html_path... with PATH_PREFIX='$PATH_PREFIX'" + if !isfile(html_path) + @info " - HTML file $html_path does not exist; skipping" + return + end + html_content = read(html_path, String) + if occursin("colab.research.google.com", html_content) + @info " - colab link already present; skipping" + return + end + # The line to edit looks like this: + #
+ # We want to insert two new list items at the end of the ul. + lines = split(html_content, '\n') + new_lines = map(lines) do line + if occursin(r"^
", line) + insertion = ( + "
  • Download notebook
    • " * + "
  • Open in Colab
    • " + ) + return replace(line, r"" => "$insertion") + else + return line + end + end + new_html_content = join(new_lines, '\n') + write(html_path, new_html_content) + @info " - done." +end + +function main(args) + if length(args) == 0 + # Get the list of .qmd files from the _quarto.yml file. This conveniently also + # checks that we are at the repo root. + qmd_files = try + quarto_config = split(read("_quarto.yml", String), '\n') + qmd_files = String[] + for line in quarto_config + m = match(r"^\s*-\s*(.+\.qmd)\s*$", line) + if m !== nothing + push!(qmd_files, m.captures[1]) + end + end + qmd_files + catch e + if e isa SystemError + error("Could not find _quarto.yml; please run this script from the repo root.") + else + rethrow(e) + end + end + for file in qmd_files + # Convert qmd to ipynb + dir = "_site/" * dirname(file) + ipynb_base = replace(basename(file), r"\.qmd$" => ".ipynb") + isdir(dir) || mkpath(dir) # mkpath is essentially mkdir -p + out_ipynb_path = joinpath(dir, ipynb_base) + convert_qmd_to_ipynb(file, out_ipynb_path) + # Add a link in the corresponding html file + html_base = replace(basename(file), r"\.qmd$" => ".html") + out_html_path = joinpath(dir, html_base) + add_ipynb_link_to_html(out_html_path, out_ipynb_path) + end + elseif length(args) == 2 + in_qmd_path, out_ipynb_path = args + convert_qmd_to_ipynb(in_qmd_path, out_ipynb_path) + add_ipynb_link_to_html(replace(out_ipynb_path, r"\.ipynb$" => ".html"), out_ipynb_path) + end +end +@main diff --git a/developers/inference/implementing-samplers/index.qmd b/developers/inference/implementing-samplers/index.qmd index dd1ad604e..9eb50d185 100644 --- a/developers/inference/implementing-samplers/index.qmd +++ b/developers/inference/implementing-samplers/index.qmd @@ -201,8 +201,7 @@ That means that we're ready to implement the only thing that really matters: `Ab `AbstractMCMC.step` defines the MCMC iteration of our `MALA` given the current `MALAState`. Specifically, the signature of the function is as follows: -```{julia} -#| eval: false +```julia function AbstractMCMC.step( # The RNG to ensure reproducibility. rng::Random.AbstractRNG, @@ -225,8 +224,7 @@ Note that `AbstractMCMC.LogDensityModel` has no other purpose; it has a single f All in all, that means that the signature for our `AbstractMCMC.step` is going to be the following: -```{julia} -#| eval: false +```julia function AbstractMCMC.step( rng::Random.AbstractRNG, # `LogDensityModel` so we know we're working with LogDensityProblems.jl model. diff --git a/tutorials/coin-flipping/index.qmd b/tutorials/coin-flipping/index.qmd index dc1674cd1..d57c79181 100755 --- a/tutorials/coin-flipping/index.qmd +++ b/tutorials/coin-flipping/index.qmd @@ -1,7 +1,7 @@ --- title: "Introduction: Coin Flipping" engine: julia -aliases: +aliases: - ../00-introduction/index.html - ../00-introduction/ ---