Implement CQL ValueSet Retrieve

docs/performance/cql.md

@@ -317,6 +317,55 @@ define InInitialPopulation:
 cql/search.sh condition-ten-rare
 ```
 
+## Ten Code Search – ValueSet
+
+In this section, CQL queries for selecting patients which have conditions with one of 10 codes are analyzed. The same codes as in section Ten Code Search are used. However instead of multiple retrieve expressions, a single retrieve expression with a value set is used.
+
+### Data
+
+| Dataset | System | # Hits | Time (s) | StdDev |  Pat./s |
+|---------|--------|-------:|---------:|-------:|--------:|
+| 1M      | LEA47  |    4 k |     0.45 |  0.005 | 2.227 M |
+| 1M      | LEA47  |  954 k |     2.53 |  0.007 | 394.9 k |
+
+### CQL Query Frequent
+
+```text
+library "condition-ten-frequent-vs"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+valueset vs: 'http://fhir.org/VCL?v1=(http://snomed.info/sct)(72892002;10509002;36955009;195662009;162864005;49727002;444814009;386661006;840539006;840544004)'
+
+context Patient
+
+define InInitialPopulation:
+  exists [Condition: vs]
+```
+
+```sh
+cql/search.sh condition-ten-frequent-vs
+```
+
+### CQL Query Rare
+
+```text
+library "condition-ten-rare-vs"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+valueset vs: 'http://fhir.org/VCL?v1=(http://snomed.info/sct)(62718007;234466008;288959006;47505003;698754002;157265008;15802004;14760008;36923009;45816000)'
+
+context Patient
+
+define InInitialPopulation:
+  exists [Condition: vs]
+```
+
+```sh
+cql/search.sh condition-ten-rare-vs
+```
+
 ## All Code Search
 
 ![](cql/all-code-search-1M.png)
@@ -335,7 +384,7 @@ cql/search.sh condition-ten-rare
 | 100k-fh | LEA58  |  100 k |     0.10 |  0.001 | 1.038 M |
 | 1M      | LEA25  |  995 k |     9.66 |  0.041 | 103.5 k |
 | 1M      | LEA36  |  995 k |     5.96 |  0.015 | 167.7 k |
-| 1M      | LEA47  |  995 k |     1.02 |  0.015 | 981.3 k |
+| 1M      | LEA47  |  995 k |     0.89 |  0.011 | 1.120 M |
 | 1M      | LEA58  |  995 k |     0.60 |  0.018 | 1.674 M |
 | 1M      | LEA79  |  995 k |     0.38 |  0.003 | 2.667 M |
 | 1M      | A5N46  |  995 k |     0.44 |  0.002 | 2.260 M |
@@ -346,6 +395,20 @@ cql/search.sh condition-ten-rare
 cql/search.sh condition-all
 ```
 
+## All Code Search – ValueSet
+
+### Data
+
+| Dataset | System | # Hits | Time (s) | StdDev |  Pat./s |
+|---------|--------|-------:|---------:|-------:|--------:|
+| 1M      | LEA47  |  995 k |    37.31 |  0.387 |  26.8 k |
+
+### CQL Query
+
+```sh
+cql/search.sh condition-all
+```
+
 ## Inpatient Stress Search
 
 ![](cql/inpatient-stress-search-1M.png)

docs/performance/cql/condition-all-vs.cql

@@ -0,0 +1,10 @@
+library "condition-all-vs"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+valueset vs: 'http://fhir.org/VCL?v1=(http://snomed.info/sct)(234466008;65275009;241929008;75498004;10509002;132281000119108;706870000;67782005;65710008;195662009;7200002;300916003;26929004;271737000;198992004;74400008;195967001;225444004;24079001;49436004;287185009;87628006;6072007;35999006;60951000119105;162864005;408512008;275272006;262574004;840539006;92691004;410429000;128188000;373587001;192127007;233678006;43724002;235919008;88805009;124171000119105;431855005;431856006;433144002;278860009;1121000119107;185086009;82423001;698754002;40055000;359817006;110030002;62564004;62106007;302297009;14760008;40275004;53741008;49727002;190905008;38822007;44054006;427089005;127013003;422034002;267060006;157265008;62718007;55680006;55680006;267036007;15802004;84757009;84757009;301011002;53827007;370247008;230265002;84229001;707577004;156073000;386661006;203082005;403190006;16114001;58150001;65966004;33737001;1734006;15724005;263102004;235595009;90560007;25064002;84114007;66857006;429280009;428251008;429007001;161622006;399211009;703151001;230745008;80394007;55822004;59621000;302870006;389087006;83664006;196416002;11218009;406602003;444470001;86175003;40095003;444448004;307731004;110359009;57676002;414564002;284551006;283371005;284549007;283385000;201834006;36955009;200936003;97331000119101;370143000;36923009;427089005;254837009;363406005;171131006;414667000;237602007;314994000;90781000119102;19169002;68962001;22298006;68235000;422587007;126906006;368581000119106;47200007;424132000;254637007;1551000119108;72892002;5602001;239872002;239873007;64859006;65363002;109838007;22253000;246677007;443165006;446096008;232353008;233604007;233604007;68496003;398152000;47505003;15777000;398254007;399912005;95417003;93761005;67811000119102;1501000119109;157141000119108;236077008;87433001;45816000;713197008;197927001;271825005;267064002;69896004;47693006;30832001;298382003;367498001;403191005;94260004;128613002;91302008;448813005;448417001;770349000;76571007;27942005;36971009;254632001;449868002;267102003;221360009;76916001;44465007;70704007;248595008;43878008;230690007;86849004;840544004;162573006;239720000;403192003;427419006;127295002;79586000;288959006;68566005;444814009;249497008;56018004;39848009)'
+
+context Patient
+
+define InInitialPopulation:
+  exists [Condition: vs]

docs/performance/cql/condition-all-vs.yml

@@ -0,0 +1,5 @@
+library: cql/condition-all-vs.cql
+group:
+- type: Patient
+  population:
+  - expression: InInitialPopulation

docs/performance/cql/condition-ten-frequent-vs.cql

@@ -0,0 +1,10 @@
+library "condition-ten-frequent-vs"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+valueset vs: 'http://fhir.org/VCL?v1=(http://snomed.info/sct)(72892002;10509002;36955009;195662009;162864005;49727002;444814009;386661006;840539006;840544004)'
+
+context Patient
+
+define InInitialPopulation:
+  exists [Condition: vs]

docs/performance/cql/condition-ten-frequent-vs.yml

@@ -0,0 +1,5 @@
+library: cql/condition-ten-frequent-vs.cql
+group:
+- type: Patient
+  population:
+  - expression: InInitialPopulation

docs/performance/cql/condition-ten-rare-vs.cql

@@ -0,0 +1,10 @@
+library "condition-ten-rare-vs"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+valueset vs: 'http://fhir.org/VCL?v1=(http://snomed.info/sct)(62718007;234466008;288959006;47505003;698754002;157265008;15802004;14760008;36923009;45816000)'
+
+context Patient
+
+define InInitialPopulation:
+  exists [Condition: vs]

docs/performance/cql/condition-ten-rare-vs.yml

@@ -0,0 +1,5 @@
+library: cql/condition-ten-rare-vs.cql
+group:
+- type: Patient
+  population:
+  - expression: InInitialPopulation

docs/performance/fhir-search.md

@@ -187,8 +187,6 @@ The script `multiple-codes-search-vs.sh` is used.
 
 ### Downloading Resources
 
-![](fhir-search/multiple-code-search-download-1M.png)
-
 | System | Dataset | Codes | # Hits | Time (s) | StdDev | Res/s ¹ |
 |--------|---------|-------|-------:|---------:|-------:|--------:|
 | A5N46  | 1M      | 10    | 29.1 M |   286.60 | 14.831 | 101.4 k |

docs/terminology-service/fhir.md

@@ -8,6 +8,7 @@ The following filters are supported.
 
 | Property | Operators           | Values     |
 |----------|---------------------|------------|
+| concept  | exists              | true/false |
 | concept  | is-a, descendent-of | code       |
 | parent   | exists              | true/false |
 | parent   | =                   | code       |

modules/cql/src/blaze/elm/compiler/clinical_operators.clj

@@ -10,7 +10,7 @@
    [blaze.elm.compiler.macros :refer [reify-expr]]
    [blaze.elm.concept :as concept]
    [blaze.elm.protocols :as p]
-   [blaze.elm.value-set :as value-set]))
+   [blaze.elm.value-set :as vs]))
 
 ;; 23.3. CalculateAge
 ;;
@@ -93,9 +93,9 @@
       (if-some [code (core/-eval code context resource scope)]
         (let [value-set (core/-eval value-set context resource scope)]
           (cond
-            (string? code) (value-set/contains-string? value-set code)
-            (code/code? code) (value-set/contains-code? value-set code)
-            (concept/concept? code) (value-set/contains-concept? value-set code)))
+            (string? code) (vs/contains-string? value-set code)
+            (code/code? code) (vs/contains-code? value-set code)
+            (concept/concept? code) (vs/contains-concept? value-set code)))
         false))
     (-form [_]
       (list 'in-value-set (core/-form code) (core/-form value-set)))))

modules/cql/src/blaze/elm/compiler/clinical_values.clj

@@ -12,7 +12,7 @@
    [blaze.elm.date-time :as date-time]
    [blaze.elm.quantity :refer [quantity]]
    [blaze.elm.ratio :refer [ratio]]
-   [blaze.elm.value-set :as value-set]))
+   [blaze.elm.value-set :as vs]))
 
 ;; 3.1. Code
 (defmethod core/compile* :elm.compiler.type/code
@@ -128,5 +128,5 @@
   [{:keys [library terminology-service] :as context} {:keys [name]}]
   ;; TODO: look into other libraries (:libraryName)
   (if-let [{:keys [id]} (find-value-set-def library name)]
-    (value-set/value-set terminology-service id)
+    (vs/value-set terminology-service id)
     (value-set-not-found-anom name context)))

modules/cql/src/blaze/elm/compiler/external_data.clj

@@ -16,6 +16,7 @@
    [blaze.elm.resource :as cr]
    [blaze.elm.spec]
    [blaze.elm.util :as elm-util]
+   [blaze.elm.value-set :as vs]
    [blaze.fhir.spec.references :as fsr]
    [blaze.util :refer [str]]
    [prometheus.alpha :as prom :refer [defcounter]]))
@@ -30,19 +31,15 @@
 (defn- code->clause-value [{:keys [system code]}]
   (str system "|" code))
 
-(defn- code-expr
-  "Returns an expression which, when evaluated, returns all resources of type
-  `data-type` which have a code equivalent to `code` at `property` and are
-  reachable through `eval-context`.
-
-  Example:
-  * data-type - \"Observation\"
-  * eval-context - \"Patient\"
-  * property - \"code\"
-  * codes - [(code \"http://loinc.org\" nil \"39156-5\")]"
-  [node eval-context data-type property codes]
-  (let [clauses [(into [property] (map code->clause-value) codes)]
-        type-query (ac/join (d/compile-type-query node data-type clauses))
+(defn- codes-clause [property codes]
+  (into [property] (map code->clause-value) codes))
+
+(defn- value-set-clause [property value-set]
+  [(str property ":in") (vs/url value-set)])
+
+(defn- compartment-query-expr
+  [node eval-context data-type clauses]
+  (let [type-query (ac/join (d/compile-type-query node data-type clauses))
         compartment-query (ac/join (d/compile-compartment-query node eval-context
                                                                 data-type clauses))]
     (reify-expr core/Expression
@@ -134,54 +131,73 @@
       (list 'retrieve (core/-form related-context-expr) data-type (d/query-clauses query)))))
 
 (defn- related-context-expr
-  [node context-expr data-type code-property codes]
-  (if (seq codes)
+  [node context-expr data-type code-property codes-expr]
+  (cond
+    (or (vs/value-set? codes-expr) (seq codes-expr))
     (if-let [result-type-name (:result-type-name (meta context-expr))]
       (let [[value-type-ns context-type] (elm-util/parse-qualified-name result-type-name)]
         (if (= "http://hl7.org/fhir" value-type-ns)
-          (let [clauses [(into [code-property] (map code->clause-value) codes)]
+          (let [clauses [(if (vs/value-set? codes-expr)
+                           (value-set-clause code-property codes-expr)
+                           (codes-clause code-property codes-expr))]
                 query (ac/join (d/compile-compartment-query node context-type data-type clauses))]
             (related-context-expr-with-codes context-expr data-type query))
           (throw-anom (unsupported-type-ns-anom value-type-ns))))
       (throw-anom unsupported-related-context-expr-without-type-anom))
+
+    :else
     (related-context-expr-without-codes node context-expr data-type)))
 
-(defn- unfiltered-context-expr [node data-type code-property codes]
-  (if (empty? codes)
+(defn- type-query-expr [node data-type clauses]
+  (let [query (ac/join (d/compile-type-query node data-type clauses))]
+    (reify-expr core/Expression
+      (-eval [_ {:keys [db]} _ _]
+        (prom/inc! retrieve-total)
+        (coll/eduction (cr/resource-mapper db) (d/execute-query db query)))
+      (-form [_]
+        `(~'retrieve ~data-type ~(d/query-clauses query))))))
+
+(defn- unfiltered-context-expr [node data-type code-property codes-expr]
+  (cond
+    (vs/value-set? codes-expr)
+    (type-query-expr node data-type [(value-set-clause code-property codes-expr)])
+
+    (empty? codes-expr)
     (reify-expr core/Expression
       (-eval [_ {:keys [db]} _ _]
         (prom/inc! retrieve-total)
         (coll/eduction (cr/resource-mapper db) (d/type-list db data-type)))
       (-form [_]
         `(~'retrieve ~data-type)))
-    (let [clauses [(into [code-property] (map code->clause-value) codes)]]
-      (let [query (ac/join (d/compile-type-query node data-type clauses))]
-        (reify-expr core/Expression
-          (-eval [_ {:keys [db]} _ _]
-            (prom/inc! retrieve-total)
-            (coll/eduction (cr/resource-mapper db) (d/execute-query db query)))
-          (-form [_]
-            `(~'retrieve ~data-type ~(d/query-clauses query))))))))
-
-(defn- expr* [node eval-context data-type code-property codes]
-  (if (empty? codes)
+
+    :else
+    (type-query-expr node data-type [(codes-clause code-property codes-expr)])))
+
+(defn- expr* [node eval-context data-type code-property codes-expr]
+  (cond
+    (vs/value-set? codes-expr)
+    (compartment-query-expr node eval-context data-type [(value-set-clause code-property codes-expr)])
+
+    (empty? codes-expr)
     (if (= data-type eval-context)
       resource-expr
       (context-expr node eval-context data-type))
-    (code-expr node eval-context data-type code-property codes)))
+
+    :else
+    (compartment-query-expr node eval-context data-type [(codes-clause code-property codes-expr)])))
 
 ;; 11.1. Retrieve
 (defn- expr
-  [{:keys [node eval-context]} context-expr data-type code-property codes]
+  [{:keys [node eval-context]} context-expr data-type code-property codes-expr]
   (cond
     context-expr
-    (related-context-expr node context-expr data-type code-property codes)
+    (related-context-expr node context-expr data-type code-property codes-expr)
 
     (= "Unfiltered" eval-context)
-    (unfiltered-context-expr node data-type code-property codes)
+    (unfiltered-context-expr node data-type code-property codes-expr)
 
     :else
-    (expr* node eval-context data-type code-property codes)))
+    (expr* node eval-context data-type code-property codes-expr)))
 
 (defn- unsupported-dynamic-codes-expr-anom [codes-expr]
   (ba/unsupported
@@ -195,7 +211,8 @@
 
 (defn- compile-codes-expr [context codes-expr]
   (let [codes-expr (core/compile* context codes-expr)]
-    (if (and (sequential? codes-expr) (every? code? codes-expr))
+    (if (or (and (sequential? codes-expr) (every? code? codes-expr))
+            (vs/value-set? codes-expr))
       codes-expr
       (throw-anom (unsupported-dynamic-codes-expr-anom codes-expr)))))