Merge pull request #121 from ImagingDataCommons/idc_v21

Pathology conversion status. Final v21 commit

Author: bcli4d

bq/generate_tables_and_views/all_joined.py

@@ -161,7 +161,7 @@ def create_all_joined(client):
     view = bigquery.Table(view_id)
     view.view_query = f"""
 -- SELECT af.*, ac.source, ac.Class, ac.Access, ac.metadata_sunset, ac.dev_bucket, ac.pub_gcs_bucket, ac.pub_aws_bucket
-SELECT af.*, ac.source, ac.Type, ac.Access, ac.metadata_sunset, ac.pub_gcs_bucket, ac.pub_aws_bucket
+SELECT af.*, ac.source, ac.Type, ac.Access, ac.metadata_sunset, ac.dev_bucket, ac.pub_gcs_bucket, ac.pub_aws_bucket
 FROM `{settings.DEV_PROJECT}.{settings.BQ_DEV_INT_DATASET}.all_flattened` af
 JOIN `{settings.DEV_PROJECT}.{settings.BQ_DEV_INT_DATASET}.all_collections` ac
 ON af.source_doi = ac.source_doi 

bq/generate_tables_and_views/idc_source_data_files.py

@@ -28,7 +28,6 @@
     parser.add_argument('--version', default=settings.CURRENT_VERSION, help='IDC version number')
     parser.add_argument('--src_project', default="idc-pdp-staging", help='Project from which tables are copied')
     parser.add_argument('--trg_project', default="nci-idc-bigquery-data", help='Project to which tables are copied')
-    # parser.add_argument('--pub_project', default="bigquery-public-data", help='Project where public datasets live')
     parser.add_argument('--pub_project', default="nci-idc-bigquery-data", help='Project where public datasets live')
     parser.add_argument('--table_ids', default={}, help="Copy all tables/views unless this is non-empty.")
     parser.add_argument('--clinical_table_ids', default={}, help="Copy all tables/views unless this is non-empty")