From 03c0612476d93818e3cba1d590efaf5373bcb0f4 Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 1 Jun 2021 15:43:48 +1000 Subject: [PATCH 1/4] [code load] Enable utf-8 as the standard for reading code from files to unify across win, linux, os x --- myst_nb/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index c9b2ebb2..d93d843d 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -187,7 +187,7 @@ def read_cell_metadata(token, cell_index): return metadata -def load_code_from_file(nb_path, file_name, token, body_lines): +def load_code_from_file(nb_path, file_name, token, body_lines, encoding="utf-8"): """load source code from a file.""" if nb_path is None: raise LoadFileParsingError("path to notebook not supplied for :load:") @@ -200,7 +200,7 @@ def load_code_from_file(nb_path, file_name, token, body_lines): ) LOGGER.warning(msg) try: - body_lines = file_path.read_text().split("\n") + body_lines = file_path.read_text(encoding=encoding).split("\n") except Exception: raise LoadFileParsingError("Can't read file from :load: {}".format(file_path)) return body_lines From d3135a6db949628966a4a182b5d6888ddc912a0e Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 1 Jun 2021 15:51:16 +1000 Subject: [PATCH 2/4] update docs --- docs/use/markdown.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/use/markdown.md b/docs/use/markdown.md index c4442abf..ca24aca4 100644 --- a/docs/use/markdown.md +++ b/docs/use/markdown.md @@ -116,7 +116,8 @@ you specify a `load` metadata attribute such as: ```` ```{warning} -This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive. +This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive. The +default encoding is set to `utf-8` when reading files for all platforms. ``` ### Syntax for markdown From 8a5c99e89ff766cff94f355d05233ebca1e8bd27 Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 1 Jun 2021 16:26:24 +1000 Subject: [PATCH 3/4] update test and test fixtures --- tests/notebooks/mystnb_codecell_file.py | 7 +++++++ .../test_codecell_file.ipynb | 16 ++++++++++++---- .../test_mystnb_features/test_codecell_file.xml | 6 ++++++ .../test_codecell_file_warnings.ipynb | 16 ++++++++++++---- .../test_codecell_file_warnings.xml | 6 ++++++ 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/tests/notebooks/mystnb_codecell_file.py b/tests/notebooks/mystnb_codecell_file.py index efc9904d..9aa0c492 100644 --- a/tests/notebooks/mystnb_codecell_file.py +++ b/tests/notebooks/mystnb_codecell_file.py @@ -1,3 +1,10 @@ # flake8: noqa import numpy as np + +# Test: Unicode + + +def c_star(x, β, γ): + + return (1 - β ** (1 / γ)) * x diff --git a/tests/test_mystnb_features/test_codecell_file.ipynb b/tests/test_mystnb_features/test_codecell_file.ipynb index b89804b9..1eab7ae5 100644 --- a/tests/test_mystnb_features/test_codecell_file.ipynb +++ b/tests/test_mystnb_features/test_codecell_file.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "a62bc462", "metadata": {}, "source": [ "# a title" @@ -10,14 +11,21 @@ { "cell_type": "code", "execution_count": 1, + "id": "2c528c38", "metadata": { - "file": "mystnb_codecell_file.py" + "load": "mystnb_codecell_file.py" }, "outputs": [], "source": [ "# flake8: noqa\n", "\n", - "import numpy as np\n" + "import numpy as np\n", + "\n", + "# Test: Unicode\n", + "\n", + "def c_star(x, β, γ):\n", + "\n", + " return (1 - β ** (1/γ)) * x\n" ] } ], @@ -44,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.8" }, "source_map": [ 11, @@ -52,5 +60,5 @@ ] }, "nbformat": 4, - "nbformat_minor": 4 + "nbformat_minor": 5 } diff --git a/tests/test_mystnb_features/test_codecell_file.xml b/tests/test_mystnb_features/test_codecell_file.xml index 545ad7b2..8f652aed 100644 --- a/tests/test_mystnb_features/test_codecell_file.xml +++ b/tests/test_mystnb_features/test_codecell_file.xml @@ -8,3 +8,9 @@ # flake8: noqa import numpy as np + + # Test: Unicode + + def c_star(x, β, γ): + + return (1 - β ** (1/γ)) * x diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb index 37f6c148..9248f583 100644 --- a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb +++ b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "f77f91c0", "metadata": {}, "source": [ "# a title" @@ -10,14 +11,21 @@ { "cell_type": "code", "execution_count": 1, + "id": "3d78b540", "metadata": { - "file": "mystnb_codecell_file.py" + "load": "mystnb_codecell_file.py" }, "outputs": [], "source": [ "# flake8: noqa\n", "\n", - "import numpy as np\n" + "import numpy as np\n", + "\n", + "# Test: Unicode\n", + "\n", + "def c_star(x, β, γ):\n", + "\n", + " return (1 - β ** (1/γ)) * x\n" ] } ], @@ -44,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" }, "source_map": [ 11, @@ -52,5 +60,5 @@ ] }, "nbformat": 4, - "nbformat_minor": 4 + "nbformat_minor": 5 } diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.xml b/tests/test_mystnb_features/test_codecell_file_warnings.xml index 5c43ddd7..4a4fb5cc 100644 --- a/tests/test_mystnb_features/test_codecell_file_warnings.xml +++ b/tests/test_mystnb_features/test_codecell_file_warnings.xml @@ -8,3 +8,9 @@ # flake8: noqa import numpy as np + + # Test: Unicode + + def c_star(x, β, γ): + + return (1 - β ** (1/γ)) * x From c234bae88cb9a8e2b45d3da13b28889abd4a000a Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 1 Jun 2021 16:35:04 +1000 Subject: [PATCH 4/4] update fixtures post pre-commit fixes to py input file --- tests/test_mystnb_features/test_codecell_file.ipynb | 7 ++++--- tests/test_mystnb_features/test_codecell_file.xml | 3 ++- .../test_mystnb_features/test_codecell_file_warnings.ipynb | 7 ++++--- tests/test_mystnb_features/test_codecell_file_warnings.xml | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_mystnb_features/test_codecell_file.ipynb b/tests/test_mystnb_features/test_codecell_file.ipynb index 1eab7ae5..8824ebba 100644 --- a/tests/test_mystnb_features/test_codecell_file.ipynb +++ b/tests/test_mystnb_features/test_codecell_file.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "a62bc462", + "id": "e67c115d", "metadata": {}, "source": [ "# a title" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "2c528c38", + "id": "18613988", "metadata": { "load": "mystnb_codecell_file.py" }, @@ -23,9 +23,10 @@ "\n", "# Test: Unicode\n", "\n", + "\n", "def c_star(x, β, γ):\n", "\n", - " return (1 - β ** (1/γ)) * x\n" + " return (1 - β ** (1 / γ)) * x\n" ] } ], diff --git a/tests/test_mystnb_features/test_codecell_file.xml b/tests/test_mystnb_features/test_codecell_file.xml index 8f652aed..e1e84b8c 100644 --- a/tests/test_mystnb_features/test_codecell_file.xml +++ b/tests/test_mystnb_features/test_codecell_file.xml @@ -11,6 +11,7 @@ # Test: Unicode + def c_star(x, β, γ): - return (1 - β ** (1/γ)) * x + return (1 - β ** (1 / γ)) * x diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb index 9248f583..ff6cf7d6 100644 --- a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb +++ b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "f77f91c0", + "id": "0d5a00cd", "metadata": {}, "source": [ "# a title" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "3d78b540", + "id": "0b392fbd", "metadata": { "load": "mystnb_codecell_file.py" }, @@ -23,9 +23,10 @@ "\n", "# Test: Unicode\n", "\n", + "\n", "def c_star(x, β, γ):\n", "\n", - " return (1 - β ** (1/γ)) * x\n" + " return (1 - β ** (1 / γ)) * x\n" ] } ], diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.xml b/tests/test_mystnb_features/test_codecell_file_warnings.xml index 4a4fb5cc..3c9cb5fb 100644 --- a/tests/test_mystnb_features/test_codecell_file_warnings.xml +++ b/tests/test_mystnb_features/test_codecell_file_warnings.xml @@ -11,6 +11,7 @@ # Test: Unicode + def c_star(x, β, γ): - return (1 - β ** (1/γ)) * x + return (1 - β ** (1 / γ)) * x