From d98dcfbbcc44a4f4c87ab8b3e054de0558aed223 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 12:58:45 +0000 Subject: [PATCH 01/12] remove contact sites; update dracopy interace --- pychunkedgraph/app/segmentation/common.py | 62 +- .../app/segmentation/legacy/routes.py | 10 - pychunkedgraph/app/segmentation/v1/routes.py | 35 - .../graph_analysis/contact_sites.py | 636 ------------------ pychunkedgraph/meshing/meshgen.py | 11 +- requirements.txt | 22 +- 6 files changed, 16 insertions(+), 760 deletions(-) delete mode 100644 pychunkedgraph/graph_analysis/contact_sites.py diff --git a/pychunkedgraph/app/segmentation/common.py b/pychunkedgraph/app/segmentation/common.py index 23157f79e..a6d1802b7 100644 --- a/pychunkedgraph/app/segmentation/common.py +++ b/pychunkedgraph/app/segmentation/common.py @@ -18,7 +18,7 @@ from pychunkedgraph.backend import history as cg_history from pychunkedgraph.backend import lineage from pychunkedgraph.backend.utils import column_keys, basetypes -from pychunkedgraph.graph_analysis import analysis, contact_sites +from pychunkedgraph.graph_analysis import analysis from pychunkedgraph.backend.graphoperation import GraphEditOperation __api_versions__ = [0, 1] @@ -847,66 +847,6 @@ def oldest_timestamp(table_id): return earliest_timestamp -### CONTACT SITES -------------------------------------------------------------- - - -def handle_contact_sites(table_id, root_id): - partners = request.args.get("partners", True, type=app_utils.toboolean) - as_list = request.args.get("as_list", True, type=app_utils.toboolean) - areas_only = request.args.get("areas_only", True, type=app_utils.toboolean) - - current_app.table_id = table_id - user_id = str(g.auth_user["id"]) - current_app.user_id = user_id - - timestamp = _parse_timestamp( - "timestamp", default_timestamp=time.time(), return_datetime=True - ) - - if "bounds" in request.args: - bounds = request.args["bounds"] - bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int - ).T - else: - bounding_box = None - - # Call ChunkedGraph - cg = app_utils.get_cg(table_id) - - cs_list, cs_metadata = contact_sites.get_contact_sites( - cg, - np.uint64(root_id), - bounding_box=bounding_box, - compute_partner=partners, - end_time=timestamp, - as_list=as_list, - areas_only=areas_only, - ) - - return cs_list, cs_metadata - - -def handle_pairwise_contact_sites(table_id, first_node_id, second_node_id): - current_app.request_type = "pairwise_contact_sites" - current_app.table_id = table_id - user_id = str(g.auth_user["id"]) - current_app.user_id = user_id - - timestamp = _parse_timestamp("timestamp", time.time(), return_datetime=True) - - exact_location = request.args.get("exact_location", True, type=app_utils.toboolean) - cg = app_utils.get_cg(table_id) - contact_sites_list, cs_metadata = contact_sites.get_contact_sites_pairwise( - cg, - np.uint64(first_node_id), - np.uint64(second_node_id), - end_time=timestamp, - exact_location=exact_location, - ) - return contact_sites_list, cs_metadata - - ### SPLIT PREVIEW -------------------------------------------------------------- diff --git a/pychunkedgraph/app/segmentation/legacy/routes.py b/pychunkedgraph/app/segmentation/legacy/routes.py index 04496f66f..fcb41641c 100644 --- a/pychunkedgraph/app/segmentation/legacy/routes.py +++ b/pychunkedgraph/app/segmentation/legacy/routes.py @@ -164,16 +164,6 @@ def handle_subgraph(table_id, root_id): return app_utils.tobinary(subgraph_result) -### CONTACT SITES -------------------------------------------------------------- - - -@bp.route("//segment//contact_sites", methods=["POST", "GET"]) -@auth_requires_permission("view") -def handle_contact_sites(table_id, root_id): - contact_sites = common.handle_contact_sites(table_id, root_id) - return jsonify(contact_sites) - - ### CHANGE LOG ----------------------------------------------------------------- diff --git a/pychunkedgraph/app/segmentation/v1/routes.py b/pychunkedgraph/app/segmentation/v1/routes.py index 8db75fa61..95d1a1d27 100644 --- a/pychunkedgraph/app/segmentation/v1/routes.py +++ b/pychunkedgraph/app/segmentation/v1/routes.py @@ -316,41 +316,6 @@ def handle_subgraph(table_id, node_id): return jsonify_with_kwargs(resp, int64_as_str=int64_as_str) -### CONTACT SITES -------------------------------------------------------------- - - -@bp.route("/table//node//contact_sites", methods=["GET"]) -@auth_requires_permission("view") -@remap_public(edit=False) -def handle_contact_sites(table_id, node_id): - int64_as_str = request.args.get("int64_as_str", default=False, type=toboolean) - contact_sites, contact_site_metadata = common.handle_contact_sites( - table_id, node_id - ) - resp = { - "contact_sites": contact_sites, - "contact_site_metadata": contact_site_metadata, - } - return jsonify_with_kwargs(resp, int64_as_str=int64_as_str) - - -@bp.route( - "/table//node/contact_sites_pair//", - methods=["GET"], -) -@auth_requires_permission("view") -@remap_public(edit=False) -def handle_pairwise_contact_sites(table_id, first_node_id, second_node_id): - int64_as_str = request.args.get("int64_as_str", default=False, type=toboolean) - contact_sites, contact_site_metadata = common.handle_pairwise_contact_sites( - table_id, first_node_id, second_node_id - ) - resp = { - "contact_sites": contact_sites, - "contact_site_metadata": contact_site_metadata, - } - return jsonify_with_kwargs(resp, int64_as_str=int64_as_str) - ### CHANGE LOG ----------------------------------------------------------------- diff --git a/pychunkedgraph/graph_analysis/contact_sites.py b/pychunkedgraph/graph_analysis/contact_sites.py deleted file mode 100644 index 93d0c05e6..000000000 --- a/pychunkedgraph/graph_analysis/contact_sites.py +++ /dev/null @@ -1,636 +0,0 @@ -import collections -from enum import auto, Enum -import itertools - -from contact_points import find_contact_points -import numpy as np - -from pychunkedgraph.backend import flatgraph_utils -from pychunkedgraph.backend.utils import column_keys - -def _get_edges_for_contact_site_graph( - cg, root_id, bounding_box, bb_is_coordinate, end_time -): - """ - Helper function for get_contact_sites that queries the ChunkedGraph for edges and returns - a list of edges that are involved in the root_id's contact sites. - """ - # Get information about the root id - # All supervoxels - sv_ids = cg.get_subgraph_nodes( - root_id, bounding_box=bounding_box, bb_is_coordinate=bb_is_coordinate - ) - # All edges that are _not_ connected / on - edges, _, areas = cg.get_subgraph_edges( - root_id, - bounding_box=bounding_box, - bb_is_coordinate=bb_is_coordinate, - connected_edges=False, - ) - # Build area lookup dictionary - edge_mask = ~np.in1d(edges, sv_ids).reshape(-1, 2) - area_mask = np.where(edge_mask)[0] - masked_areas = areas[area_mask] - contact_sites_svs = edges[edge_mask] - contact_sites_svs_area_dict = collections.defaultdict(int) - - for area, sv_id in zip(masked_areas, contact_sites_svs): - contact_sites_svs_area_dict[sv_id] += area - - # Extract svs from contacting root ids - unique_contact_sites_svs = np.unique(contact_sites_svs) - - # Load edges of these contact sites' supervoxels - edges_contact_sites_svs_rows = cg.read_node_id_rows( - node_ids=unique_contact_sites_svs, - columns=[column_keys.Connectivity.Partner, column_keys.Connectivity.Connected], - end_time=end_time, - end_time_inclusive=True, - ) - - contact_sites_edges = [] - for row_information in edges_contact_sites_svs_rows.items(): - sv_edges, _, _ = cg._retrieve_connectivity(row_information) - contact_sites_edges.extend(sv_edges) - if len(contact_sites_edges) == 0: - return None, None, False - contact_sites_edges_array = np.array(contact_sites_edges) - contact_sites_edge_mask = np.isin( - contact_sites_edges_array[:, 1], unique_contact_sites_svs - ) - # Fake edges to ensure lone contact site supervoxels show up as a connected component - self_edges = np.stack((unique_contact_sites_svs, unique_contact_sites_svs), axis=-1) - contact_sites_graph_edges = np.concatenate( - (contact_sites_edges_array[contact_sites_edge_mask], self_edges), axis=0 - ) - return contact_sites_graph_edges, contact_sites_svs_area_dict, True - -def get_contact_sites( - cg, - root_id, - bounding_box=None, - bb_is_coordinate=True, - compute_partner=True, - end_time=None, - voxel_location=True, - areas_only=False, - as_list=False -): - """ - Given a root id, return two lists: the first contains all the contact sites with other roots in the dataset, - the second is metadata specifying exactly what data the first list contains. - - If compute_partner=True, the first returned list is a list of tuples of length two. The first element of the tuple - is a contact site partner root id. The second element is a list of all the contact sites (tuples) root_id makes - with this contact partner. - - If compute_partner=False, the first returned list is a list of all the contact sites. - - The voxel_location and areas_only parameters affect the tuples in the list of contact sites mentioned above. - If voxel_location=False, then the first two entries of the tuple are chunk coordinates that - bound part of the contact site; the third entry is the area of the contact site. If voxel_location=True, - the first two entries are the positions of those two chunks in global coordinates instead. - If areas_only=True, then the tuple is just the area and no location is returned. - """ - contact_sites_graph_edges, contact_sites_svs_area_dict, any_contact_sites = _get_edges_for_contact_site_graph( - cg, root_id, bounding_box, bb_is_coordinate, end_time - ) - - if not any_contact_sites: - return collections.defaultdict(list) - - contact_sites_svs_area_dict_vec = np.vectorize(contact_sites_svs_area_dict.get) - - graph, _, _, unique_sv_ids = flatgraph_utils.build_gt_graph( - contact_sites_graph_edges, make_directed=True - ) - - connected_components = flatgraph_utils.connected_components(graph) - - contact_site_dict = collections.defaultdict(list) - # First create intermediary map of supervoxel to contact sites, so we - # can call cg.get_roots() on all supervoxels at once. - intermediary_sv_dict = {} - for cc in connected_components: - cc_sv_ids = unique_sv_ids[cc] - contact_sites_areas = contact_sites_svs_area_dict_vec(cc_sv_ids) - - representative_sv = cc_sv_ids[0] - # Tuple of location and area of contact site - chunk_coordinates = cg.get_chunk_coordinates(representative_sv) - if areas_only: - data_pair = np.sum(contact_sites_areas) - elif voxel_location: - voxel_lower_bound = ( - cg.vx_vol_bounds[:, 0] + cg.chunk_size * chunk_coordinates - ) - voxel_upper_bound = cg.vx_vol_bounds[:, 0] + cg.chunk_size * ( - chunk_coordinates + 1 - ) - data_pair = ( - voxel_lower_bound * cg.segmentation_resolution, - voxel_upper_bound * cg.segmentation_resolution, - np.sum(contact_sites_areas), - ) - else: - data_pair = ( - chunk_coordinates, - chunk_coordinates + 1, - np.sum(contact_sites_areas), - ) - - if compute_partner: - # Cast np.uint64 to int for dict key because int is hashable - intermediary_sv_dict[int(representative_sv)] = data_pair - else: - contact_site_dict[len(contact_site_dict)].append(data_pair) - if compute_partner: - sv_list = np.array(list(intermediary_sv_dict.keys()), dtype=np.uint64) - partner_roots = cg.get_roots(sv_list) - for i in range(len(partner_roots)): - contact_site_dict[int(partner_roots[i])].append( - intermediary_sv_dict.get(int(sv_list[i])) - ) - - contact_site_list = [] - for partner_id in contact_site_dict: - if compute_partner: - contact_site_list.append((np.uint64(partner_id), contact_site_dict[partner_id])) - else: - contact_site_list.append((*contact_site_dict[partner_id])) - - if compute_partner: - contact_site_metadata = ['segment id', 'lower bound coordinate', 'upper bound coordinate', 'area'] - else: - contact_site_metadata = ['lower bound coordinate', 'upper bound coordinate', 'area'] - - return contact_site_list, contact_site_metadata - - -def _retrieve_connectivity_optimized(cg, dict_item): - """ - An altered version of cg._retrieve_connectivity that is optimized - for the purpose of retrieving contact sites. - - WARNING: This method of determining connectivity is soon to be - deprecated. - 10/15/19 - """ - node_id, row = dict_item - - tmp = set() - for x in itertools.chain.from_iterable( - generation.value for generation in row[column_keys.Connectivity.Connected][::-1] - ): - tmp.remove(x) if x in tmp else tmp.add(x) - - connected_indices = np.fromiter(tmp, np.uint64) - if column_keys.Connectivity.Partner in row: - edges = np.fromiter( - itertools.chain.from_iterable( - (node_id, partner_id) - for generation in row[column_keys.Connectivity.Partner][::-1] - for partner_id in generation.value - ), - dtype=basetypes.NODE_ID, - ).reshape((-1, 2)) - edges_in = cg._connected_or_not(edges, connected_indices, True) - edges_out = cg._connected_or_not(edges, connected_indices, False) - else: - edges_in = np.empty((0, 2), basetypes.NODE_ID) - edges_out = np.empty((0, 2), basetypes.NODE_ID) - - if column_keys.Connectivity.Area in row: - areas = np.fromiter( - itertools.chain.from_iterable( - generation.value - for generation in row[column_keys.Connectivity.Area][::-1] - ), - dtype=basetypes.EDGE_AREA, - ) - areas_out = cg._connected_or_not(areas, connected_indices, False) - else: - areas_out = np.empty(0, basetypes.EDGE_AREA) - - return edges_in, edges_out, areas_out - - -def find_approximate_middle_point(points, resolution): - """ - Given a list of points and a resolution, choose a point from - the list that is approximately in the center. - """ - bbox_min = np.amin(points, axis=0) - bbox_max = np.amax(points, axis=0) - distances_from_bbox_max = np.sum(((bbox_max - points) * resolution), axis=1) - distances_from_bbox_min = np.sum(((points - bbox_min) * resolution), axis=1) - # For each point, greater distance of distance to bbox_max or bbox_min - greater_distance = np.amax( - np.vstack((distances_from_bbox_max, distances_from_bbox_min)), axis=0 - ) - return points[np.argmin(greater_distance)] - - -def _find_points_nm_coordinate(cg, chunk_coordinates, coordinate_within_chunk): - """ - Given a lvl1/lvl2 chunk coordinate and a position in the chunk, return a global - coordinate representing that voxel's position in the dataset. - """ - points_voxel_coordinate = ( - cg.get_chunk_voxel_location(chunk_coordinates) + coordinate_within_chunk - ) - return points_voxel_coordinate * cg.segmentation_resolution - - -def _get_approximate_middle_contact_point_same_chunk(cg, sv1, sv2, ws_seg): - """ - Given two supervoxels in the same chunk, return a contact point amongst - their contact points that is approximately in the center of the contact. - If there are no contact points, return a point that is - approximately in the center of sv1. - """ - chunk_coordinate = cg.get_chunk_coordinates(sv1) - contact_points = find_contact_points(ws_seg, sv1, sv2) - if len(contact_points) == 0: - sv1_locations = np.where(ws_seg == sv1) - sv1_points = np.vstack((sv1_locations[0], sv1_locations[1], sv1_locations[2])) - middle_point = find_approximate_middle_point( - sv1_points, cg.segmentation_resolution - ) - else: - middle_point = find_approximate_middle_point( - contact_points[:, 0, :], cg.segmentation_resolution - ) - return _find_points_nm_coordinate(cg, chunk_coordinate, middle_point) - - -def _get_approximate_middle_contact_point_across_chunks(cg, sv1, sv2, sv1_chunk_seg): - """ - Given two supervoxels in neighboring chunks, return a contact point amongst - their contact points that is approximately in the center of the contact. - If there are no contact points, return a point that is - approximately in the center of sv1. - """ - chunk_coordinate = cg.get_chunk_coordinates(sv1) - other_chunk_coordinate = cg.get_chunk_coordinates(sv2) - sv2_chunk_seg = cg.download_chunk_segmentation(other_chunk_coordinate) - crossing_index = np.where(chunk_coordinate - other_chunk_coordinate)[0][0] - positive_crossing = (chunk_coordinate - other_chunk_coordinate)[crossing_index] < 0 - sv1_chunk_plane_index = -1 if positive_crossing else 0 - sv2_chunk_plane_index = 0 if positive_crossing else -1 - if crossing_index == 0: - sv1_chunk_plane = sv1_chunk_seg[sv1_chunk_plane_index, :, :] - sv2_chunk_plane = sv2_chunk_seg[sv2_chunk_plane_index, :, :] - elif crossing_index == 1: - sv1_chunk_plane = sv1_chunk_seg[:, sv1_chunk_plane_index, :] - sv2_chunk_plane = sv2_chunk_seg[:, sv2_chunk_plane_index, :] - else: - sv1_chunk_plane = sv1_chunk_seg[:, :, sv1_chunk_plane_index] - sv2_chunk_plane = sv2_chunk_seg[:, :, sv2_chunk_plane_index] - chunk_border = np.stack((sv1_chunk_plane, sv2_chunk_plane), axis=crossing_index) - contact_points = find_contact_points(chunk_border, sv1, sv2) - if len(contact_points) == 0: - sv1_locations = np.where(chunk_border == sv1) - sv1_points = np.vstack((sv1_locations[0], sv1_locations[1], sv1_locations[2])) - middle_point = find_approximate_middle_point( - sv1_points, cg.segmentation_resolution - ) - else: - middle_point = find_approximate_middle_point( - contact_points[:, 0, :], cg.segmentation_resolution - ) - return _find_points_nm_coordinate(cg, chunk_coordinate, middle_point) - - -def _get_approximate_middle_sv_point(cg, sv, ws_seg): - """ - Get a point in a given supervoxel that is approximately - in its center. - """ - chunk_coordinate = cg.get_chunk_coordinates(sv) - sv_locations = np.where(ws_seg == sv) - sv_points = np.vstack((sv_locations[0], sv_locations[1], sv_locations[2])) - middle_point = find_approximate_middle_point(sv_points, cg.segmentation_resolution) - return _find_points_nm_coordinate(cg, chunk_coordinate, middle_point) - - -class EdgeType(Enum): - SameChunk = auto() - AdjacentChunks = auto() - NonAdjacentChunks = auto() - - -def _choose_contact_site_edge(cg, first_node_unconnected_edges, second_node_sv_ids): - """ - Choose an edge to represent a contact site from a list of edges. For performance reasons, - try to choose an edge that is not a cross chunk edge (to avoid downloading multiple chunks). - """ - edge_mask = np.where( - np.isin(first_node_unconnected_edges[:, 1], second_node_sv_ids) - )[0] - filtered_unconnected_edges = first_node_unconnected_edges[edge_mask] - smallest_distance = None - best_edge = None - for edge in filtered_unconnected_edges: - chunk_coordinates_first_node = cg.get_chunk_coordinates(edge[0]) - chunk_coordinates_second_node = cg.get_chunk_coordinates(edge[1]) - if np.array_equal(chunk_coordinates_first_node, chunk_coordinates_second_node): - return (edge, EdgeType.SameChunk) - distance = abs( - np.sum(chunk_coordinates_first_node) - np.sum(chunk_coordinates_second_node) - ) - if best_edge is None or distance < smallest_distance: - smallest_distance = distance - best_edge = edge - if smallest_distance == 1: - return (edge, EdgeType.AdjacentChunks) - return (edge, EdgeType.NonAdjacentChunks) - - -def _get_contact_site_edges( - cg, - first_node_unconnected_edges, - first_node_unconnected_areas, - second_node_connected_edges, - second_node_sv_ids, -): - """ - Given two sets of supervoxels, find all contact sites between the two sets, and for each - contact site return an edge that represents the site. - """ - # Retrieve edges that connect first node with second node - unconnected_edge_mask = np.where( - np.isin(first_node_unconnected_edges[:, 1], second_node_sv_ids) - )[0] - filtered_unconnected_edges = first_node_unconnected_edges[unconnected_edge_mask] - filtered_areas = first_node_unconnected_areas[unconnected_edge_mask] - contact_sites_svs_area_dict = collections.defaultdict(int) - for i in range(filtered_unconnected_edges.shape[0]): - sv_id = filtered_unconnected_edges[i, 1] - area = filtered_areas[i] - contact_sites_svs_area_dict[sv_id] += area - contact_sites_svs_area_dict_vec = np.vectorize(contact_sites_svs_area_dict.get) - unique_contact_sites_svs = np.unique(filtered_unconnected_edges[:, 1]) - # Retrieve edges that connect second node contact sites with other second node contact sites - connected_edge_test = np.isin(second_node_connected_edges, unique_contact_sites_svs) - connected_edges_mask = np.where(np.all(connected_edge_test, axis=1))[0] - filtered_connected_edges = second_node_connected_edges[connected_edges_mask] - # Make fake edges from contact site svs to themselves to make sure they appear in the created graph - self_edges = np.stack((unique_contact_sites_svs, unique_contact_sites_svs), axis=-1) - contact_sites_graph_edges = np.concatenate( - (filtered_connected_edges, self_edges), axis=0 - ) - - graph, _, _, unique_sv_ids = flatgraph_utils.build_gt_graph( - contact_sites_graph_edges, make_directed=True - ) - connected_components = flatgraph_utils.connected_components(graph) - - contact_site_edges = [] - for cc in connected_components: - cc_sv_ids = unique_sv_ids[cc] - contact_sites_areas = contact_sites_svs_area_dict_vec(cc_sv_ids) - contact_site_edge, contact_site_edge_type = _choose_contact_site_edge( - cg, filtered_unconnected_edges, cc_sv_ids - ) - contact_site_edge_info = ( - contact_site_edge, - contact_site_edge_type, - np.sum(contact_sites_areas), - ) - contact_site_edges.append(contact_site_edge_info) - return contact_site_edges - - -def _get_contact_site_edges_to_inspect( - cg, sv_ids_set1, sv_ids_set2, end_time, optimize_unsafe -): - """ - Given two sets of supervoxels, read their connectivity data from the graph, - find all contact sites between the two sets, and for each - contact site return an edge that represents the site. - """ - # Combine the supervoxel into one set to optimize the call to read_node_id_rows - all_sv_ids = np.concatenate((sv_ids_set1, sv_ids_set2)) - is_in_set1 = np.concatenate((np.ones(len(sv_ids_set1)), np.zeros(len(sv_ids_set2)))) - is_in_set1_dict = dict(zip(all_sv_ids, is_in_set1)) - - all_sv_rows = cg.read_node_id_rows( - node_ids=all_sv_ids, - columns=[ - column_keys.Connectivity.Area, - column_keys.Connectivity.Partner, - column_keys.Connectivity.Connected, - ], - end_time=end_time, - end_time_inclusive=True, - ) - - # Retrieve the connectivity data - edges_in1 = [] - edges_out1 = [] - areas_out1 = [] - edges_in2 = [] - edges_out2 = [] - areas_out2 = [] - for row_information in all_sv_rows.items(): - sv_id, _ = row_information - if optimize_unsafe: - # WARNING: We are deprecating the old way of retrieving connectivity so _retrieve_connectivity_optimized - # will soon have to be updated - 10/15/2019 - sv_edges_in, sv_edges_out, sv_areas_out = _retrieve_connectivity_optimized( - cg, row_information - ) - else: - sv_edges_in, _, _ = cg._retrieve_connectivity( - row_information, connected_edges=True - ) - sv_edges_out, _, sv_areas_out = cg._retrieve_connectivity( - row_information, connected_edges=False - ) - if is_in_set1_dict[sv_id]: - edges_in1.append(sv_edges_in) - edges_out1.append(sv_edges_out) - areas_out1.append(sv_areas_out) - else: - edges_in2.append(sv_edges_in) - edges_out2.append(sv_edges_out) - areas_out2.append(sv_areas_out) - - def _concat_nonempty(x): - if len(x) > 0: - return np.concatenate(x) - return x - - edges_in1 = _concat_nonempty(edges_in1) - edges_out1 = _concat_nonempty(edges_out1) - areas_out1 = _concat_nonempty(areas_out1) - edges_in2 = _concat_nonempty(edges_in2) - edges_out2 = _concat_nonempty(edges_out2) - areas_out2 = _concat_nonempty(areas_out2) - - - # Check the number of contact sites from both sides. The number may be different; the higher - # number is always more accurate. - contact_sites_edges = _get_contact_site_edges( - cg, edges_out1, areas_out1, edges_in2, sv_ids_set2 - ) - contact_sites_edges_other_direction = _get_contact_site_edges( - cg, edges_out2, areas_out2, edges_in1, sv_ids_set1 - ) - - if len(contact_sites_edges_other_direction) > len(contact_sites_edges): - edges_to_inspect = contact_sites_edges_other_direction - else: - edges_to_inspect = contact_sites_edges - - return edges_to_inspect - - -def _get_sv_contact_site_candidates(cg, first_node_id, second_node_id): - """ - Given two node ids, return their two sets of children supervoxels - that could possibly have a contact point with a supervoxel of the - other node id. - """ - # Get lvl2 ids of both roots - first_node_l2_ids = cg.get_children_at_layer(first_node_id, 2) - second_node_l2_ids = cg.get_children_at_layer(second_node_id, 2) - - if len(first_node_l2_ids) > len(second_node_l2_ids): - smaller_set_l2_ids = second_node_l2_ids - bigger_set_l2_ids = first_node_l2_ids - else: - smaller_set_l2_ids = first_node_l2_ids - bigger_set_l2_ids = second_node_l2_ids - - # Make a dict holding all the chunk coordinates of all the lvl2 ids of the "smaller" root, and the coordinates of the neighboring chunks - chunk_coordinate_dict = collections.defaultdict(set) - for i in range(len(smaller_set_l2_ids)): - l2_id = smaller_set_l2_ids[i] - chunk_coordinates = cg.get_chunk_coordinates(l2_id) - chunk_coordinate_dict[tuple(chunk_coordinates)].add(i) - for j in range(3): - adjacent_chunk_coordinates = np.copy(chunk_coordinates) - adjacent_chunk_coordinates[j] += 1 - chunk_coordinate_dict[tuple(adjacent_chunk_coordinates)].add(i) - adjacent_chunk_coordinates[j] -= 2 - chunk_coordinate_dict[tuple(adjacent_chunk_coordinates)].add(i) - - candidate_smaller_set_l2_ids = set() - candidate_bigger_set_l2_ids = [] - - # For the "bigger" root id, filter out all lvl2 ids that do not appear in the chunk_coordinate_dict - for l2_id in bigger_set_l2_ids: - chunk_coordinates = cg.get_chunk_coordinates(l2_id) - new_candidate_smaller_set_l2_ids = chunk_coordinate_dict.get( - tuple(chunk_coordinates) - ) - if new_candidate_smaller_set_l2_ids is not None: - for new_candidate in new_candidate_smaller_set_l2_ids: - candidate_smaller_set_l2_ids.add(smaller_set_l2_ids[new_candidate]) - candidate_bigger_set_l2_ids.append(l2_id) - - # Get the supervoxel of the filtered lvl2 ids to retrieve their edges - sv_ids_set1 = cg.get_children(list(candidate_smaller_set_l2_ids), flatten=True) - sv_ids_set2 = cg.get_children(candidate_bigger_set_l2_ids, flatten=True) - - return (sv_ids_set1, sv_ids_set2) - - -def _get_exact_contact_sites(cg, edges_to_inspect): - """ - Given a list of edges between two supervoxels, for each edge get a global - coordinate in the dataset where the two supervoxels contact. - """ - chunk_coordinate_list = [] - for edge_to_inspect in edges_to_inspect: - edge = edge_to_inspect[0] - chunk_coordinate_list.append(tuple(cg.get_chunk_coordinates(edge[0]))) - chunk_coordinate_array = np.array( - chunk_coordinate_list, dtype=[("x", "i8"), ("y", "i8"), ("z", "i8")] - ) - sorted_chunk_coordinates_indices = np.argsort( - chunk_coordinate_array, order=("x", "y", "z") - ) - - memoized_chunk = None - last_chunk_coordinate = None - contact_sites = [] - for index in sorted_chunk_coordinates_indices: - edge, edgeType, area = edges_to_inspect[index] - cur_chunk_coordinate = chunk_coordinate_array[index] - if ( - last_chunk_coordinate is None - or last_chunk_coordinate != cur_chunk_coordinate - ): - last_chunk_coordinate = cur_chunk_coordinate - memoized_chunk = cg.download_chunk_segmentation( - np.array( - ( - cur_chunk_coordinate[0], - cur_chunk_coordinate[1], - cur_chunk_coordinate[2], - ) - ) - ) - if edgeType == EdgeType.SameChunk: - contact_point = _get_approximate_middle_contact_point_same_chunk( - cg, edge[0], edge[1], memoized_chunk - ) - elif edgeType == EdgeType.AdjacentChunks: - contact_point = _get_approximate_middle_contact_point_across_chunks( - cg, edge[0], edge[1], memoized_chunk - ) - else: - contact_point = _get_approximate_middle_sv_point( - cg, edge[0], memoized_chunk - ) - contact_sites.append((contact_point, area)) - return contact_sites - - -def get_contact_sites_pairwise( - cg, - first_node_id, - second_node_id, - end_time=None, - exact_location=True, - optimize_unsafe=False, -): - """ - Given two node ids, find the locations and areas of their contact sites in the dataset. - This function returns two lists, the first representing the contact sites, and the second - is metadata specifying the exact data contained in the first list. - - If exact_location=True, the first return value is a list of tuples of two elements, - where the first element in the tuple is a global coordinate in the dataset, and - the second is an area. - - If exact_location=False, the first return value is a list of tuples of three elements. - The first and second elements are the global coordinates that the contact site appears somewhere - between, and the third is an area. - """ - sv_ids_set1, sv_ids_set2 = _get_sv_contact_site_candidates( - cg, first_node_id, second_node_id - ) - if len(sv_ids_set1) == 0 or len(sv_ids_set2) == 0: - return [] - edges_to_inspect = _get_contact_site_edges_to_inspect( - cg, sv_ids_set1, sv_ids_set2, end_time, optimize_unsafe - ) - if exact_location: - contact_sites = _get_exact_contact_sites(cg, edges_to_inspect) - contact_site_metadata = ['coordinate', 'area'] - else: - contact_sites = [] - contact_site_metadata = ['lower_bound_coordinate', 'upper_bound_coordinate', 'area'] - for edge_to_inspect in edges_to_inspect: - edge, _, area = edge_to_inspect - contact_sites.append( - ( - cg.get_chunk_voxel_location(cg.get_chunk_coordinates(edge[0])) * cg.segmentation_resolution, - cg.get_chunk_voxel_location(cg.get_chunk_coordinates(edge[0]) + 1) * cg.segmentation_resolution, - area, - ) - ) - return contact_sites, contact_site_metadata diff --git a/pychunkedgraph/meshing/meshgen.py b/pychunkedgraph/meshing/meshgen.py index f5aac3da9..ec858b3e5 100644 --- a/pychunkedgraph/meshing/meshgen.py +++ b/pychunkedgraph/meshing/meshgen.py @@ -33,13 +33,12 @@ def decode_draco_mesh_buffer(fragment): try: - mesh_object = DracoPy.decode(fragment) + mesh_object = DracoPy.decode_buffer_to_mesh(fragment) vertices = np.array(mesh_object.points) faces = np.array(mesh_object.faces) except ValueError: raise ValueError("Not a valid draco mesh") - assert vertices.size % 3 == 0, "Draco mesh vertices not 3-D" num_vertices = len(vertices) # For now, just return this dict until we figure out @@ -1017,9 +1016,9 @@ def chunk_mesh_task_new_remapping( mesh.vertices[:] += chunk_offset if encoding == "draco": try: - file_contents = DracoPy.encode( - mesh.vertices, - mesh.faces, + file_contents = DracoPy.encode_mesh_to_buffer( + mesh.vertices.flatten("C"), + mesh.faces.flatten("C"), **draco_encoding_settings, ) except: @@ -1206,7 +1205,7 @@ def chunk_mesh_task_new_remapping( ) try: - new_fragment_b = DracoPy.encode( + new_fragment_b = DracoPy.encode_mesh_to_buffer( new_fragment["vertices"], new_fragment["faces"], **draco_encoding_options, diff --git a/requirements.txt b/requirements.txt index 67e12a697..de5790ccc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,20 @@ -importlib-metadata==4.13.0 -flask click >= 8.0 -cloud-files>=1.25.2 +cloud-files>=4.21.1 requests>=2.25.0 grpcio>=1.36.1 certifi>=2020.12.5 +networkx>=2.1 +google-cloud-bigtable>=0.33.0 +google-cloud-datastore>=1.8 +middle-auth-client>=3.11.0 +dracopy==1.3.0 +zmesh>=1.7.0 +fastremap>=1.14.0 +jsonschema<4.0 pyopenssl +flask numpy pandas -networkx==2.1 -google-cloud-bigtable>=0.33.0 -google-cloud-datastore>=1.8<=2.0dev flask_cors codecov multiwrapper @@ -18,13 +22,7 @@ python-json-logger zstandard redis rq -middle-auth-client>=3.11.0 -dracopy~=1.0.0 -zmesh -fastremap -contact-points messagingclient -jsonschema<4.0 cloud-volume pyyaml datastoreflex From d2bb5b8d3c01c4f47beb1f4ac27b43cb9ecfc971 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 13:04:59 +0000 Subject: [PATCH 02/12] pin cv to match main version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index de5790ccc..b93a6454d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ dracopy==1.3.0 zmesh>=1.7.0 fastremap>=1.14.0 jsonschema<4.0 +cloud-volume==8.26.0 pyopenssl flask numpy @@ -23,6 +24,5 @@ zstandard redis rq messagingclient -cloud-volume pyyaml datastoreflex From c62cf0a71002e3bd59d0b4f2b93fdfc28ee3ddb9 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 13:11:35 +0000 Subject: [PATCH 03/12] remove pip upgrade --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c26ffdc1b..c06921cce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,5 @@ FROM seunglab/pychunkedgraph:graph-tool_dracopy COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY requirements.txt /app -RUN pip install --upgrade pip \ - && pip install --no-cache-dir --upgrade -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt COPY . /app \ No newline at end of file From 28513f531fe64b1eb75d2e1b06d096ad5d873687 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 13:14:39 +0000 Subject: [PATCH 04/12] update base image --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c06921cce..f7c79d6e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -# FROM gcr.io/neuromancer-seung-import/pychunkedgraph:graph-tool_dracopy -FROM seunglab/pychunkedgraph:graph-tool_dracopy +FROM caveconnectome/pychunkedgraph:base_042124 COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY requirements.txt /app From 011cbdf5d695c160b660fead6c6747f8d65b9865 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 13:19:56 +0000 Subject: [PATCH 05/12] remove gcr references from cloudbuild --- cloudbuild.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 275ebc266..21f4cc58d 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,31 +1,25 @@ steps: + # Login to Docker Hub - name: "gcr.io/cloud-builders/docker" entrypoint: "bash" args: ["-c", "docker login --username=$$USERNAME --password=$$PASSWORD"] secretEnv: ["USERNAME", "PASSWORD"] - # - name: 'gcr.io/cloud-builders/docker' - # args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/pychunkedgraph', '.' ] + - name: "gcr.io/cloud-builders/docker" entrypoint: "bash" args: - "-c" - | - docker build -t gcr.io/$PROJECT_ID/pychunkedgraph:$TAG_NAME . + docker build -t $$USERNAME/pychunkedgraph:$TAG_NAME . timeout: 600s - - name: "gcr.io/cloud-builders/docker" - entrypoint: "bash" - args: - [ - "-c", - "docker tag gcr.io/$PROJECT_ID/pychunkedgraph:$TAG_NAME $$USERNAME/pychunkedgraph:$TAG_NAME", - ] secretEnv: ["USERNAME"] + + # Push the final image to Dockerhub - name: "gcr.io/cloud-builders/docker" entrypoint: "bash" args: ["-c", "docker push $$USERNAME/pychunkedgraph:$TAG_NAME"] secretEnv: ["USERNAME"] -images: - - "gcr.io/$PROJECT_ID/pychunkedgraph:$TAG_NAME" + availableSecrets: secretManager: - versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1 From 9a5f93cecd96895a02689b9df4f881b94935e403 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 25 Mar 2025 06:40:37 -0700 Subject: [PATCH 06/12] fixing tox --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 41a22d4e6..7c0a6a5f0 100644 --- a/tox.ini +++ b/tox.ini @@ -12,3 +12,4 @@ deps = pytest>=7.0 numpy commands = python -m pytest {posargs} ./pychunkedgraph/tests/ install_command = {toxinidir}/tox_install_command.sh {opts} {packages} +allowlist_externals = ./tox_install_command.sh \ No newline at end of file From e93a1a4365619324c9dfada7952aee2f54f9a392 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 25 Mar 2025 06:43:21 -0700 Subject: [PATCH 07/12] tox fix try 2 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7c0a6a5f0..ad6d64f1e 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,4 @@ deps = pytest>=7.0 numpy commands = python -m pytest {posargs} ./pychunkedgraph/tests/ install_command = {toxinidir}/tox_install_command.sh {opts} {packages} -allowlist_externals = ./tox_install_command.sh \ No newline at end of file +allowlist_externals = {toxinidir}/tox_install_command.sh \ No newline at end of file From 3a92816e5c977290fa5c03123a24cd8630417d08 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 21:17:49 +0000 Subject: [PATCH 08/12] tox py311 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ad6d64f1e..9dd380d3a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37dev +envlist = py311 [testenv] setenv = HOME = {env:HOME} From 183a100ef59013296018e093c510c34f4f19bcaa Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 21:22:18 +0000 Subject: [PATCH 09/12] tox remove old pip --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 9dd380d3a..1b24a0e98 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,4 @@ deps = pytest>=7.0 pytest-mock pytest-timeout numpy -commands = python -m pytest {posargs} ./pychunkedgraph/tests/ -install_command = {toxinidir}/tox_install_command.sh {opts} {packages} -allowlist_externals = {toxinidir}/tox_install_command.sh \ No newline at end of file +commands = python -m pytest {posargs} ./pychunkedgraph/tests/ \ No newline at end of file From 4dd2e4a5450c7ff01db4e9df03ebde91fa03c8a5 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 21:31:05 +0000 Subject: [PATCH 10/12] fix: change np.int and np.bool to int and bool; no longer supported --- pychunkedgraph/app/app_utils.py | 2 +- pychunkedgraph/app/meshing/common.py | 2 +- pychunkedgraph/app/segmentation/common.py | 8 +-- pychunkedgraph/backend/__init__.py | 6 +- pychunkedgraph/backend/chunkedgraph.py | 56 +++++++++---------- pychunkedgraph/backend/chunkedgraph_edits.py | 10 ++-- pychunkedgraph/backend/chunks/utils.py | 2 +- pychunkedgraph/backend/cutting.py | 2 +- pychunkedgraph/backend/cutting_test.py | 2 +- pychunkedgraph/backend/graphoperation.py | 32 +++++------ pychunkedgraph/backend/history.py | 2 +- .../benchmarking/graph_measurements.py | 14 ++--- pychunkedgraph/benchmarking/timings.py | 8 +-- pychunkedgraph/creator/chunkcreator.py | 12 ++-- pychunkedgraph/creator/data_test.py | 2 +- pychunkedgraph/edge_gen/edgetask.py | 6 +- pychunkedgraph/exporting/export.py | 14 ++--- pychunkedgraph/ingest/cluster.py | 2 +- pychunkedgraph/ingest/ran_agglomeration.py | 10 ++-- pychunkedgraph/meshing/mesh_io.py | 2 +- pychunkedgraph/meshing/meshengine.py | 20 +++---- pychunkedgraph/meshing/meshgen.py | 12 ++-- pychunkedgraph/meshing/meshgen_utils.py | 2 +- pychunkedgraph/rechunking/transformer.py | 6 +- pychunkedgraph/tests/helpers.py | 2 +- pychunkedgraph/tests/test_uncategorized.py | 6 +- 26 files changed, 121 insertions(+), 121 deletions(-) diff --git a/pychunkedgraph/app/app_utils.py b/pychunkedgraph/app/app_utils.py index 9b6da4cf8..1eadcb1df 100644 --- a/pychunkedgraph/app/app_utils.py +++ b/pychunkedgraph/app/app_utils.py @@ -250,7 +250,7 @@ def ccs(coordinates_nm_): ccs = [np.array(list(cc)) for cc in nx.connected_components(graph)] return ccs - coordinates = np.array(coordinates, dtype=np.int) + coordinates = np.array(coordinates, dtype=int) coordinates_nm = coordinates * cg.cv.resolution node_ids = np.array(node_ids, dtype=np.uint64) diff --git a/pychunkedgraph/app/meshing/common.py b/pychunkedgraph/app/meshing/common.py index b3da3ef7c..ec7e35f09 100644 --- a/pychunkedgraph/app/meshing/common.py +++ b/pychunkedgraph/app/meshing/common.py @@ -72,7 +72,7 @@ def handle_get_manifest(table_id, node_id): if "bounds" in request.args: bounds = request.args["bounds"] bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int + [b.split("-") for b in bounds.split("_")], dtype=int ).T else: bounding_box = None diff --git a/pychunkedgraph/app/segmentation/common.py b/pychunkedgraph/app/segmentation/common.py index a6d1802b7..376d7f4cb 100644 --- a/pychunkedgraph/app/segmentation/common.py +++ b/pychunkedgraph/app/segmentation/common.py @@ -516,7 +516,7 @@ def handle_leaves(table_id, root_id): if "bounds" in request.args: bounds = request.args["bounds"] bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int + [b.split("-") for b in bounds.split("_")], dtype=int ).T else: bounding_box = None @@ -559,7 +559,7 @@ def handle_leaves_many(table_id): if "bounds" in request.args: bounds = request.args["bounds"] bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int + [b.split("-") for b in bounds.split("_")], dtype=int ).T else: bounding_box = None @@ -591,7 +591,7 @@ def handle_leaves_from_leave(table_id, atomic_id): if "bounds" in request.args: bounds = request.args["bounds"] bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int + [b.split("-") for b in bounds.split("_")], dtype=int ).T else: bounding_box = None @@ -618,7 +618,7 @@ def handle_subgraph(table_id, root_id): if "bounds" in request.args: bounds = request.args["bounds"] bounding_box = np.array( - [b.split("-") for b in bounds.split("_")], dtype=np.int + [b.split("-") for b in bounds.split("_")], dtype=int ).T else: bounding_box = None diff --git a/pychunkedgraph/backend/__init__.py b/pychunkedgraph/backend/__init__.py index 4ac4d99c3..b31d27209 100644 --- a/pychunkedgraph/backend/__init__.py +++ b/pychunkedgraph/backend/__init__.py @@ -75,14 +75,14 @@ def bigtable_config(self): @property def resolution(self) -> np.ndarray: - return np.array(self._ws_cv.resolution) + return np.array(self._ws_cv.resolution) @property def layer_count(self) -> int: if self._layer_count: return self._layer_count bbox = np.array(self._ws_cv.bounds.to_list()).reshape(2, 3) - n_chunks = ((bbox[1] - bbox[0]) / self._graph_config.chunk_size).astype(np.int) + n_chunks = ((bbox[1] - bbox[0]) / self._graph_config.chunk_size).astype(int) self._layer_count = ( int(np.ceil(log_n(np.max(n_chunks), self._graph_config.fanout))) + 2 ) @@ -102,7 +102,7 @@ def layer_chunk_bounds(self) -> Dict: layer_bounds_d = {} for layer in range(2, self.layer_count): layer_bounds = chunks_boundary / (2 ** (layer - 2)) - layer_bounds_d[layer] = np.ceil(layer_bounds).astype(np.int) + layer_bounds_d[layer] = np.ceil(layer_bounds).astype(int) self._layer_bounds_d = layer_bounds_d return self._layer_bounds_d diff --git a/pychunkedgraph/backend/chunkedgraph.py b/pychunkedgraph/backend/chunkedgraph.py index 7096b6f9f..568acc4c7 100644 --- a/pychunkedgraph/backend/chunkedgraph.py +++ b/pychunkedgraph/backend/chunkedgraph.py @@ -471,10 +471,10 @@ def get_serialized_info(self): return info def adjust_vol_coordinates_to_cv( - self, x: np.int, y: np.int, z: np.int, resolution: Sequence[np.int] + self, x: int, y: int, z: int, resolution: Sequence[int] ): resolution = np.array(resolution) - scaling = np.array(self.cv.resolution / resolution, dtype=np.int) + scaling = np.array(self.cv.resolution / resolution, dtype=int) x = x / scaling[0] - self.vx_vol_bounds[0, 0] y = y / scaling[1] - self.vx_vol_bounds[1, 0] @@ -484,25 +484,25 @@ def adjust_vol_coordinates_to_cv( def get_chunk_coordinates_from_vol_coordinates( self, - x: np.int, - y: np.int, - z: np.int, - resolution: Sequence[np.int], + x: int, + y: int, + z: int, + resolution: Sequence[int], ceil: bool = False, layer: int = 1, ) -> np.ndarray: """Translates volume coordinates to chunk_coordinates - :param x: np.int - :param y: np.int - :param z: np.int + :param x: int + :param y: int + :param z: int :param resolution: np.ndarray :param ceil bool :param layer: int :return: """ resolution = np.array(resolution) - scaling = np.array(self.cv.resolution / resolution, dtype=np.int) + scaling = np.array(self.cv.resolution / resolution, dtype=int) x = (x / scaling[0] - self.vx_vol_bounds[0, 0]) / self.chunk_size[0] y = (y / scaling[1] - self.vx_vol_bounds[1, 0]) / self.chunk_size[1] @@ -516,7 +516,7 @@ def get_chunk_coordinates_from_vol_coordinates( if ceil: coords = np.ceil(coords) - return coords.astype(np.int) + return coords.astype(int) def get_chunk_layer(self, node_or_chunk_id: np.uint64) -> int: """Extract Layer from Node ID or Chunk ID @@ -533,7 +533,7 @@ def get_chunk_layers(self, node_or_chunk_ids: Sequence[np.uint64]) -> np.ndarray :return: np.ndarray """ if len(node_or_chunk_ids) == 0: - return np.array([], dtype=np.int) + return np.array([], dtype=int) return self._get_chunk_layer_vec(node_or_chunk_ids) @@ -611,7 +611,7 @@ def get_chunk_ids_from_node_ids(self, node_ids: Iterable[np.uint64]) -> np.ndarr :return: np.ndarray(dtype=np.uint64) """ if len(node_ids) == 0: - return np.array([], dtype=np.int) + return np.array([], dtype=int) return self._get_chunk_id_vec(node_ids) @@ -957,9 +957,9 @@ def get_cross_chunk_edges_layer(self, cross_edges): :return: array of length n """ if len(cross_edges) == 0: - return np.array([], dtype=np.int) + return np.array([], dtype=int) - cross_chunk_edge_layers = np.ones(len(cross_edges), dtype=np.int) + cross_chunk_edge_layers = np.ones(len(cross_edges), dtype=int) cross_edge_coordinates = [] for cross_edge in cross_edges: @@ -970,7 +970,7 @@ def get_cross_chunk_edges_layer(self, cross_edges): ] ) - cross_edge_coordinates = np.array(cross_edge_coordinates, dtype=np.int) + cross_edge_coordinates = np.array(cross_edge_coordinates, dtype=int) for layer in range(2, self.n_layers): edge_diff = np.sum( @@ -2123,7 +2123,7 @@ def add_atomic_edges_in_chunks( partners = np.concatenate([connected_ids, disconnected_ids]) affinities = np.concatenate([connected_affs, disconnected_affs]) areas = np.concatenate([connected_areas, disconnected_areas]) - connected = np.arange(len(connected_ids), dtype=np.int) + connected = np.arange(len(connected_ids), dtype=int) val_dict = { column_keys.Connectivity.Partner: partners, @@ -2276,7 +2276,7 @@ def _read_subchunks_thread(chunk_coord): max_child_ids = max_child_ids[sorting] counter = collections.defaultdict(int) - max_child_ids_occ_so_far = np.zeros(len(max_child_ids), dtype=np.int) + max_child_ids_occ_so_far = np.zeros(len(max_child_ids), dtype=int) for i_row in range(len(max_child_ids)): max_child_ids_occ_so_far[i_row] = counter[max_child_ids[i_row]] counter[max_child_ids[i_row]] += 1 @@ -2470,7 +2470,7 @@ def _write_out_connected_components(args) -> None: if n_jobs > 0: spacing = np.linspace( 0, len(atomic_partner_id_dict_keys), n_jobs + 1 - ).astype(np.int) + ).astype(int) starts = spacing[:-1] ends = spacing[1:] @@ -2523,7 +2523,7 @@ def _write_out_connected_components(args) -> None: n_jobs = np.min([n_jobs, len(ccs)]) - spacing = np.linspace(0, len(ccs), n_jobs + 1).astype(np.int) + spacing = np.linspace(0, len(ccs), n_jobs + 1).astype(int) starts = spacing[:-1] ends = spacing[1:] @@ -2775,7 +2775,7 @@ def get_roots( time_stamp = get_google_compatible_time_stamp(time_stamp, round_up=False) stop_layer = self.n_layers if not stop_layer else min(self.n_layers, stop_layer) - layer_mask = np.ones(len(node_ids), dtype=np.bool) + layer_mask = np.ones(len(node_ids), dtype=bool) for _ in range(n_tries): layer_mask[self.get_chunk_layers(node_ids) >= stop_layer] = False @@ -3528,7 +3528,7 @@ def normalize_bounding_box( if bounding_box is None: return None - bbox = np.array(bounding_box, dtype=np.int) + bbox = np.array(bounding_box, dtype=int) if bb_is_coordinate: bbox[0] = self.get_chunk_coordinates_from_vol_coordinates( bbox[0][0], @@ -4018,7 +4018,7 @@ def _connected_or_not(self, array, connected_indices, connected): indices or their complement. Used to select edge descriptors for those that are either connected or not connected. """ - mask = np.zeros((array.shape[0],), dtype=np.bool) + mask = np.zeros((array.shape[0],), dtype=bool) mask[connected_indices] = True if connected: @@ -4271,8 +4271,8 @@ def _run_multicut( # Get edges between local supervoxels n_chunks_affected = np.product( - (np.ceil(bounding_box[1] / self.chunk_size)).astype(np.int) - - (np.floor(bounding_box[0] / self.chunk_size)).astype(np.int) + (np.ceil(bounding_box[1] / self.chunk_size)).astype(int) + - (np.floor(bounding_box[0] / self.chunk_size)).astype(int) ) self.logger.debug("Number of affected chunks: %d" % n_chunks_affected) @@ -4387,7 +4387,7 @@ def get_chunk_voxel_location(self, chunk_coordinate): offset = 0 else: offset = self.vx_vol_bounds[:, 0] - return np.array((offset + self.chunk_size * chunk_coordinate), dtype=np.int) + return np.array((offset + self.chunk_size * chunk_coordinate), dtype=int) def download_chunk_segmentation(self, chunk_coordinate): """ @@ -4441,9 +4441,9 @@ def mask_nodes_by_bounding_box( self, nodes: Union[Iterable[np.uint64], np.uint64], bounding_box: Optional[Sequence[Sequence[int]]] = None, - ) -> Iterable[np.bool]: + ) -> Iterable[bool]: if bounding_box is None: - return np.ones(len(nodes), np.bool) + return np.ones(len(nodes), bool) else: chunk_coordinates = np.array([self.get_chunk_coordinates(c) for c in nodes]) layers = self.get_chunk_layers(nodes) diff --git a/pychunkedgraph/backend/chunkedgraph_edits.py b/pychunkedgraph/backend/chunkedgraph_edits.py index 5f42d9b99..e909adaf5 100644 --- a/pychunkedgraph/backend/chunkedgraph_edits.py +++ b/pychunkedgraph/backend/chunkedgraph_edits.py @@ -384,7 +384,7 @@ def old_parent_childrens(eh, node_ids, layer): :param eh: EditHelper instance :param node_ids: list of np.uint64s - :param layer: np.int + :param layer: int :return: """ assert len(node_ids) > 0 @@ -429,7 +429,7 @@ def compute_cross_chunk_connected_components(eh, node_ids, layer): :param eh: EditHelper :param node_ids: list of np.uint64s - :param layer: np.int + :param layer: int :return: """ assert len(node_ids) > 0 @@ -720,7 +720,7 @@ def get_root(self, node_id, get_all_parents=False): return parents[-1] def get_layer_children(self, node_id, layer, layer_only=False): - """ Get + """ Get :param node_id: :param layer: @@ -757,7 +757,7 @@ def get_layer_parent(self, node_id, layer, layer_only=False, """ Gets parent in particular layer :param node_id: np.uint64 - :param layer: np.int + :param layer: int :param layer_only: bool :param choose_lower_layer: bool :return: @@ -810,7 +810,7 @@ def get_old_node_ids(self, node_id, layer): """ Acquires old node ids for new node id :param node_id: np.uint64 - :param layer: np.int + :param layer: int :return: """ lower_old_node_ids = self._get_lower_old_node_ids(node_id) diff --git a/pychunkedgraph/backend/chunks/utils.py b/pychunkedgraph/backend/chunks/utils.py index db3665839..6f1095c90 100644 --- a/pychunkedgraph/backend/chunks/utils.py +++ b/pychunkedgraph/backend/chunks/utils.py @@ -3,7 +3,7 @@ def get_chunks_boundary(voxel_boundary, chunk_size) -> np.ndarray: """returns number of chunks in each dimension""" - return np.ceil((voxel_boundary / chunk_size)).astype(np.int) + return np.ceil((voxel_boundary / chunk_size)).astype(int) def compute_chunk_id( diff --git a/pychunkedgraph/backend/cutting.py b/pychunkedgraph/backend/cutting.py index c960c959d..da81a3ee4 100644 --- a/pychunkedgraph/backend/cutting.py +++ b/pychunkedgraph/backend/cutting.py @@ -344,7 +344,7 @@ def _gt_mincut_sanity_check(self, partition): for i_cc in np.unique(partition.a): # Make sure to read real ids and not graph ids cc_list = self.unique_supervoxel_ids[ - np.array(np.where(partition.a == i_cc)[0], dtype=np.int) + np.array(np.where(partition.a == i_cc)[0], dtype=int) ] if np.any(np.in1d(self.sources, cc_list)): diff --git a/pychunkedgraph/backend/cutting_test.py b/pychunkedgraph/backend/cutting_test.py index 4007872d4..b40a451e2 100644 --- a/pychunkedgraph/backend/cutting_test.py +++ b/pychunkedgraph/backend/cutting_test.py @@ -42,7 +42,7 @@ def ex_graph(): edgelist = np.array(edgelist) - edges = edgelist[:, :2].astype(np.int) - 1 + edges = edgelist[:, :2].astype(int) - 1 weights = edgelist[:, 2].astype(np.float) n_nodes = 100000 diff --git a/pychunkedgraph/backend/graphoperation.py b/pychunkedgraph/backend/graphoperation.py index 894177092..91f053f4d 100644 --- a/pychunkedgraph/backend/graphoperation.py +++ b/pychunkedgraph/backend/graphoperation.py @@ -25,8 +25,8 @@ def __init__( cg: "ChunkedGraph", *, user_id: str, - source_coords: Optional[Sequence[Sequence[np.int]]] = None, - sink_coords: Optional[Sequence[Sequence[np.int]]] = None, + source_coords: Optional[Sequence[Sequence[int]]] = None, + sink_coords: Optional[Sequence[Sequence[int]]] = None, ) -> None: super().__init__() self.cg = cg @@ -374,9 +374,9 @@ class MergeOperation(GraphEditOperation): :param added_edges: Supervoxel IDs of all added edges [[source, sink]] :type added_edges: Sequence[Sequence[np.uint64]] :param source_coords: world space coordinates in nm, corresponding to IDs in added_edges[:,0], defaults to None - :type source_coords: Optional[Sequence[Sequence[np.int]]], optional + :type source_coords: Optional[Sequence[Sequence[int]]], optional :param sink_coords: world space coordinates in nm, corresponding to IDs in added_edges[:,1], defaults to None - :type sink_coords: Optional[Sequence[Sequence[np.int]]], optional + :type sink_coords: Optional[Sequence[Sequence[int]]], optional :param affinities: edge weights for newly added edges, entries corresponding to added_edges, defaults to None :type affinities: Optional[Sequence[np.float32]], optional """ @@ -389,8 +389,8 @@ def __init__( *, user_id: str, added_edges: Sequence[Sequence[np.uint64]], - source_coords: Optional[Sequence[Sequence[np.int]]] = None, - sink_coords: Optional[Sequence[Sequence[np.int]]] = None, + source_coords: Optional[Sequence[Sequence[int]]] = None, + sink_coords: Optional[Sequence[Sequence[int]]] = None, affinities: Optional[Sequence[np.float32]] = None, ) -> None: super().__init__(cg, user_id=user_id, source_coords=source_coords, sink_coords=sink_coords) @@ -466,10 +466,10 @@ class SplitOperation(GraphEditOperation): :type removed_edges: Sequence[Sequence[np.uint64]] :param source_coords: world space coordinates in nm, corresponding to IDs in removed_edges[:,0], defaults to None - :type source_coords: Optional[Sequence[Sequence[np.int]]], optional + :type source_coords: Optional[Sequence[Sequence[int]]], optional :param sink_coords: world space coordinates in nm, corresponding to IDs in removed_edges[:,1], defaults to None - :type sink_coords: Optional[Sequence[Sequence[np.int]]], optional + :type sink_coords: Optional[Sequence[Sequence[int]]], optional """ __slots__ = ["removed_edges"] @@ -480,8 +480,8 @@ def __init__( *, user_id: str, removed_edges: Sequence[Sequence[np.uint64]], - source_coords: Optional[Sequence[Sequence[np.int]]] = None, - sink_coords: Optional[Sequence[Sequence[np.int]]] = None, + source_coords: Optional[Sequence[Sequence[int]]] = None, + sink_coords: Optional[Sequence[Sequence[int]]] = None, ) -> None: super().__init__(cg, user_id=user_id, source_coords=source_coords, sink_coords=sink_coords) self.removed_edges = np.atleast_2d(removed_edges).astype(basetypes.NODE_ID) @@ -553,12 +553,12 @@ class MulticutOperation(GraphEditOperation): :param sink_ids: Supervoxel IDs that should be separated from supervoxel IDs in source_ids :type sink_ids: Sequence[np.uint64] :param source_coords: world space coordinates in nm, corresponding to IDs in source_ids - :type source_coords: Sequence[Sequence[np.int]] + :type source_coords: Sequence[Sequence[int]] :param sink_coords: world space coordinates in nm, corresponding to IDs in sink_ids - :type sink_coords: Sequence[Sequence[np.int]] + :type sink_coords: Sequence[Sequence[int]] :param bbox_offset: Padding for min-cut bounding box, applied to min/max coordinates retrieved from source_coords and sink_coords, defaults to None - :type bbox_offset: Sequence[np.int] + :type bbox_offset: Sequence[int] """ __slots__ = ["source_ids", "sink_ids", "removed_edges", "bbox_offset"] @@ -570,9 +570,9 @@ def __init__( user_id: str, source_ids: Sequence[np.uint64], sink_ids: Sequence[np.uint64], - source_coords: Sequence[Sequence[np.int]], - sink_coords: Sequence[Sequence[np.int]], - bbox_offset: Sequence[np.int], + source_coords: Sequence[Sequence[int]], + sink_coords: Sequence[Sequence[int]], + bbox_offset: Sequence[int], ) -> None: super().__init__(cg, user_id=user_id, source_coords=source_coords, sink_coords=sink_coords) self.removed_edges = None # Calculated from coordinates and IDs diff --git a/pychunkedgraph/backend/history.py b/pychunkedgraph/backend/history.py index ee621a1dd..ed9301891 100644 --- a/pychunkedgraph/backend/history.py +++ b/pychunkedgraph/backend/history.py @@ -296,7 +296,7 @@ def past_operation_ids(self, root_id=None): ancs.extend(nx.algorithms.dag.ancestors(self.lineage_graph, root_id)) if len(ancs) == 0: - return np.array([], dtype=np.int) + return np.array([], dtype=int) ancs = fastremap.unique(np.array(ancs, dtype=np.uint64)) diff --git a/pychunkedgraph/benchmarking/graph_measurements.py b/pychunkedgraph/benchmarking/graph_measurements.py index a526c5012..939e9702d 100644 --- a/pychunkedgraph/benchmarking/graph_measurements.py +++ b/pychunkedgraph/benchmarking/graph_measurements.py @@ -20,10 +20,10 @@ def count_nodes_and_edges(table_id, n_threads=1): bounds = np.array(cg.cv.bounds.to_list()).reshape(2, -1).T bounds -= bounds[:, 0:1] - chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(np.int) + chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(int) chunk_coord_gen = itertools.product(*[range(*r) for r in chunk_id_bounds]) - chunk_coords = np.array(list(chunk_coord_gen), dtype=np.int) + chunk_coords = np.array(list(chunk_coord_gen), dtype=int) order = np.arange(len(chunk_coords)) np.random.shuffle(order) @@ -88,10 +88,10 @@ def count_and_download_nodes(table_id, save_dir=f"{HOME}/benchmarks/", bounds = np.array(cg.cv.bounds.to_list()).reshape(2, -1).T bounds -= bounds[:, 0:1] - chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(np.int) + chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(int) chunk_coord_gen = itertools.product(*[range(*r) for r in chunk_id_bounds]) - chunk_coords = np.array(list(chunk_coord_gen), dtype=np.int) + chunk_coords = np.array(list(chunk_coord_gen), dtype=int) order = np.arange(len(chunk_coords)) np.random.shuffle(order) @@ -272,10 +272,10 @@ def get_merge_candidates(table_id, save_dir=f"{HOME}/benchmarks/", bounds = np.array(cg.cv.bounds.to_list()).reshape(2, -1).T bounds -= bounds[:, 0:1] - chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(np.int) + chunk_id_bounds = np.ceil((bounds / cg.chunk_size[:, None])).astype(int) chunk_coord_gen = itertools.product(*[range(*r) for r in chunk_id_bounds]) - chunk_coords = np.array(list(chunk_coord_gen), dtype=np.int) + chunk_coords = np.array(list(chunk_coord_gen), dtype=int) order = np.arange(len(chunk_coords)) np.random.shuffle(order) @@ -343,7 +343,7 @@ def _get_merge_candidates(args): if len(edges) == 0: continue - + edges = np.sort(np.array(edges), axis=1) cols = {"sv1": edges[:, 0], "sv2": edges[:, 1], "parent": ps} diff --git a/pychunkedgraph/benchmarking/timings.py b/pychunkedgraph/benchmarking/timings.py index d9dd603cc..654f57d51 100644 --- a/pychunkedgraph/benchmarking/timings.py +++ b/pychunkedgraph/benchmarking/timings.py @@ -188,7 +188,7 @@ def get_root_timings(table_id, save_dir=f"{HOME}/benchmarks/", job_size=500, del cg_serialized_info["credentials"] time_start = time.time() - np.random.seed(np.int(time.time())) + np.random.seed(int(time.time())) if len(rep_l1_nodes) < job_size * 64 * 3 * 10: replace = True @@ -295,7 +295,7 @@ def get_subgraph_timings(table_id, save_dir=f"{HOME}/benchmarks/", job_size=500, time_start = time.time() order = np.arange(len(n_l1_nodes_per_root)) - np.random.seed(np.int(time.time())) + np.random.seed(int(time.time())) if len(order) < job_size * 64 * 3 * 10: replace = True @@ -354,7 +354,7 @@ def _get_subgraph_timings(args): timings = [] for root_id, rep_l1_chunk_id in zip(root_ids, rep_l1_chunk_ids): - bb = np.array([rep_l1_chunk_id, rep_l1_chunk_id + 1], dtype=np.int) + bb = np.array([rep_l1_chunk_id, rep_l1_chunk_id + 1], dtype=int) time_start = time.time() sv_ids = cg.get_subgraph_nodes(root_id, bb, bb_is_coordinate=False) @@ -411,7 +411,7 @@ def get_merge_split_timings(table_id, save_dir=f"{HOME}/benchmarks/", job_size=5 time_start = time.time() order = np.arange(len(merge_edges)) - np.random.seed(np.int(time.time())) + np.random.seed(int(time.time())) replace = False diff --git a/pychunkedgraph/creator/chunkcreator.py b/pychunkedgraph/creator/chunkcreator.py index 58a038160..3699a581b 100644 --- a/pychunkedgraph/creator/chunkcreator.py +++ b/pychunkedgraph/creator/chunkcreator.py @@ -280,7 +280,7 @@ def create_chunked_graph(table_id=None, cv_url=None, ws_url=None, fan_out=2, print("\n\n\n --- LAYER %d --- \n\n\n" % layer_id) parent_chunk_ids = child_chunk_ids // cg.fan_out - parent_chunk_ids = parent_chunk_ids.astype(np.int) + parent_chunk_ids = parent_chunk_ids.astype(int) u_pcids, inds = np.unique(parent_chunk_ids, axis=0, return_inverse=True) @@ -293,7 +293,7 @@ def create_chunked_graph(table_id=None, cv_url=None, ws_url=None, fan_out=2, multi_args = [] for ind in range(len(u_pcids)): multi_args.append([table_id, layer_id, - child_chunk_ids[inds == ind].astype(np.int), + child_chunk_ids[inds == ind].astype(int), n_threads_per_process]) child_chunk_ids = u_pcids @@ -341,7 +341,7 @@ def _preprocess_chunkedgraph_data_thread(args): file_name = os.path.basename(fp).split(".")[0] # Read coordinates from file path - x1, x2, y1, y2, z1, z2 = np.array(re.findall("[\d]+", file_name), dtype=np.int)[:6] + x1, x2, y1, y2, z1, z2 = np.array(re.findall("[\d]+", file_name), dtype=int)[:6] if np.any((bbox[0] - np.array([x2, y2, z2])) >= 0) or \ np.any((bbox[1] - np.array([x1, y1, z1])) <= 0): @@ -361,15 +361,15 @@ def _preprocess_chunkedgraph_data_thread(args): s_c = np.where(d == gap)[0] chunk_coord = c.copy() - chunk1_id = np.array(chunk_coord / chunk_size, dtype=np.int) + chunk1_id = np.array(chunk_coord / chunk_size, dtype=int) chunk_coord[s_c] += chunk_size[s_c] - chunk2_id = np.array(chunk_coord / chunk_size, dtype=np.int) + chunk2_id = np.array(chunk_coord / chunk_size, dtype=int) between_chunk_ids = np.concatenate([between_chunk_ids, np.array([chunk1_id, chunk2_id])[None]]) between_chunk_paths = np.concatenate([between_chunk_paths, [fp]]) else: - chunk_coord = np.array(c / chunk_size, dtype=np.int) + chunk_coord = np.array(c / chunk_size, dtype=int) if "disconnected" in file_name: in_chunk_disconnected_ids = np.concatenate([in_chunk_disconnected_ids, chunk_coord[None]]) diff --git a/pychunkedgraph/creator/data_test.py b/pychunkedgraph/creator/data_test.py index 48af7b51b..519b6079d 100644 --- a/pychunkedgraph/creator/data_test.py +++ b/pychunkedgraph/creator/data_test.py @@ -17,7 +17,7 @@ def _test_unique_edge_assignment_thread(args): ids = creator_utils.read_edge_file_h5(path)["node_ids"] u_ids = np.unique(ids) - u_id_d = dict(zip(u_ids, np.ones(len(u_ids), dtype=np.int))) + u_id_d = dict(zip(u_ids, np.ones(len(u_ids), dtype=int))) add_counter = collections.Counter(u_id_d) id_dict += add_counter diff --git a/pychunkedgraph/edge_gen/edgetask.py b/pychunkedgraph/edge_gen/edgetask.py index 478b03434..511984e2b 100644 --- a/pychunkedgraph/edge_gen/edgetask.py +++ b/pychunkedgraph/edge_gen/edgetask.py @@ -541,9 +541,9 @@ def run_task_bundle(settings: Mapping, roi: Tuple[slice, slice, slice]): settings["regiongraph"]["regiongraph_path_output"]) regiongraph_chunksize = tuple(settings["regiongraph"]["chunksize"]) - chunkgraph_chunksize = np.array(cgraph.chunk_size, dtype=np.int) - output_watershed_chunksize = np.array(watershed_output.underlying, dtype=np.int) - outer_chunksize = np.maximum(chunkgraph_chunksize, output_watershed_chunksize, dtype=np.int) + chunkgraph_chunksize = np.array(cgraph.chunk_size, dtype=int) + output_watershed_chunksize = np.array(watershed_output.underlying, dtype=int) + outer_chunksize = np.maximum(chunkgraph_chunksize, output_watershed_chunksize, dtype=int) # Iterate through TaskBundle using a minimal chunk size that is a multiple # of the output watershed chunk size and the Chunked Graph chunk size. diff --git a/pychunkedgraph/exporting/export.py b/pychunkedgraph/exporting/export.py index 5f9c629a0..eedbbd4d3 100644 --- a/pychunkedgraph/exporting/export.py +++ b/pychunkedgraph/exporting/export.py @@ -17,14 +17,14 @@ def get_sv_to_root_id_mapping_chunk(cg, chunk_coords, vol=None): """ sv_to_root_mapping = {} - chunk_coords = np.array(chunk_coords, dtype=np.int) + chunk_coords = np.array(chunk_coords, dtype=int) if np.any((chunk_coords % cg.chunk_size) != 0): raise Exception("Chunk coords have to match a chunk corner exactly") chunk_coords = chunk_coords / cg.chunk_size - chunk_coords = chunk_coords.astype(np.int) - bb = np.array([chunk_coords, chunk_coords + 1], dtype=np.int) + chunk_coords = chunk_coords.astype(int) + bb = np.array([chunk_coords, chunk_coords + 1], dtype=int) remapped_vol = None vol_shape = None @@ -122,13 +122,13 @@ def write_flat_segmentation(cg, dataset_name, bounding_box=None, block_factor=2, dataset_bounding_box = np.array(from_cv.bounds.to_list()) block_bounding_box_cg = \ - [np.floor(dataset_bounding_box[:3] / cg.chunk_size).astype(np.int), - np.ceil(dataset_bounding_box[3:] / cg.chunk_size).astype(np.int)] + [np.floor(dataset_bounding_box[:3] / cg.chunk_size).astype(int), + np.ceil(dataset_bounding_box[3:] / cg.chunk_size).astype(int)] if bounding_box is not None: bounding_box_cg = \ - [np.floor(bounding_box[0] / cg.chunk_size).astype(np.int), - np.ceil(bounding_box[1] / cg.chunk_size).astype(np.int)] + [np.floor(bounding_box[0] / cg.chunk_size).astype(int), + np.ceil(bounding_box[1] / cg.chunk_size).astype(int)] m = block_bounding_box_cg[0] < bounding_box_cg[0] block_bounding_box_cg[0][m] = bounding_box_cg[0][m] diff --git a/pychunkedgraph/ingest/cluster.py b/pychunkedgraph/ingest/cluster.py index 933f7adbe..2ab8ea15d 100644 --- a/pychunkedgraph/ingest/cluster.py +++ b/pychunkedgraph/ingest/cluster.py @@ -69,7 +69,7 @@ def _create_atomic_chunk(im_info: str, coord: Sequence[int]): from .ingestion import create_atomic_chunk_helper imanager = IngestionManager.from_pickle(im_info) - coord = np.array(list(coord), dtype=np.int) + coord = np.array(list(coord), dtype=int) create_atomic_chunk_helper(ChunkTask(imanager.cg_meta, coord), imanager) _post_task_completion(imanager, 2, coord) diff --git a/pychunkedgraph/ingest/ran_agglomeration.py b/pychunkedgraph/ingest/ran_agglomeration.py index 1e90fa3c4..556a43932 100644 --- a/pychunkedgraph/ingest/ran_agglomeration.py +++ b/pychunkedgraph/ingest/ran_agglomeration.py @@ -115,7 +115,7 @@ def _collect_edge_data(imanager, chunk_coord): for d in [-1, 1]: for dim in range(3): - diff = np.zeros([3], dtype=np.int) + diff = np.zeros([3], dtype=int) diff[dim] = d adjacent_chunk_coord = chunk_coord + diff x, y, z = adjacent_chunk_coord @@ -217,7 +217,7 @@ def define_active_edges(edge_dict, mapping) -> Union[Dict, np.ndarray]: agg_id_1 = mapping_vec(edge_dict[k].node_ids1) else: assert len(edge_dict[k].node_ids2) == 0 - active[k] = np.array([], dtype=np.bool) + active[k] = np.array([], dtype=bool) continue agg_id_2 = mapping_vec(edge_dict[k].node_ids2) @@ -245,19 +245,19 @@ def read_raw_agglomeration_data(imanager, chunk_coord: np.ndarray): filenames = [] for mip_level in range(0, int(imanager.cg_meta.layer_count - 1)): - x, y, z = np.array(chunk_coord / 2 ** mip_level, dtype=np.int) + x, y, z = np.array(chunk_coord / 2 ** mip_level, dtype=int) filenames.append(f"done_{mip_level}_{x}_{y}_{z}_{chunk_id}.data.zst") for d in [-1, 1]: for dim in range(3): - diff = np.zeros([3], dtype=np.int) + diff = np.zeros([3], dtype=int) diff[dim] = d adjacent_chunk_coord = chunk_coord + diff x, y, z = adjacent_chunk_coord adjacent_chunk_id = compute_chunk_id(layer=1, x=x, y=y, z=z) for mip_level in range(0, int(imanager.cg_meta.layer_count - 1)): - x, y, z = np.array(adjacent_chunk_coord / 2 ** mip_level, dtype=np.int) + x, y, z = np.array(adjacent_chunk_coord / 2 ** mip_level, dtype=int) filenames.append( f"done_{mip_level}_{x}_{y}_{z}_{adjacent_chunk_id}.data.zst" ) diff --git a/pychunkedgraph/meshing/mesh_io.py b/pychunkedgraph/meshing/mesh_io.py index 282ff0b97..40c02bba0 100644 --- a/pychunkedgraph/meshing/mesh_io.py +++ b/pychunkedgraph/meshing/mesh_io.py @@ -167,7 +167,7 @@ def load_obj(self): norms.append(0) faces.append(face) - self._faces = np.array(faces, dtype=np.int) - 1 + self._faces = np.array(faces, dtype=int) - 1 self._vertices = np.array(vertices, dtype=np.float) self._normals = np.array(normals, dtype=np.float) diff --git a/pychunkedgraph/meshing/meshengine.py b/pychunkedgraph/meshing/meshengine.py index fd2a461c3..da25244c3 100644 --- a/pychunkedgraph/meshing/meshengine.py +++ b/pychunkedgraph/meshing/meshengine.py @@ -85,7 +85,7 @@ def mesh_multiple_layers(self, layers=None, bounding_box=None, if layers is None: layers = range(1, int(self.cg.n_layers + 1)) - layers = np.array(layers, dtype=np.int) + layers = np.array(layers, dtype=int) layers = layers[layers > 0] layers = layers[layers < self.highest_mesh_layer + 1] @@ -106,16 +106,16 @@ def mesh_single_layer(self, layer, bounding_box=None, block_factor=2, block_bounding_box_cg = \ [np.floor(dataset_bounding_box[:3] / - self.cg.chunk_size).astype(np.int), + self.cg.chunk_size).astype(int), np.ceil(dataset_bounding_box[3:] / - self.cg.chunk_size).astype(np.int)] + self.cg.chunk_size).astype(int)] if bounding_box is not None: bounding_box_cg = \ [np.floor(bounding_box[0] / - self.cg.chunk_size).astype(np.int), + self.cg.chunk_size).astype(int), np.ceil(bounding_box[1] / - self.cg.chunk_size).astype(np.int)] + self.cg.chunk_size).astype(int)] m = block_bounding_box_cg[0] < bounding_box_cg[0] block_bounding_box_cg[0][m] = bounding_box_cg[0][m] @@ -147,7 +147,7 @@ def mesh_single_layer(self, layer, bounding_box=None, block_factor=2, block_bounding_box_cg[1][2], block_factor)) - blocks = np.array(list(block_iter), dtype=np.int) + blocks = np.array(list(block_iter), dtype=int) cg_info = self.cg.get_serialized_info() del (cg_info['credentials']) @@ -176,11 +176,11 @@ def mesh_single_layer(self, layer, bounding_box=None, block_factor=2, def create_manifests_for_higher_layers(self, n_threads=1): root_id_max = self.cg.get_max_node_id( - self.cg.get_chunk_id(layer=np.int(self.cg.n_layers), - x=np.int(0), y=np.int(0), - z=np.int(0))) + self.cg.get_chunk_id(layer=int(self.cg.n_layers), + x=int(0), y=int(0), + z=int(0))) - root_id_blocks = np.linspace(1, root_id_max, n_threads*3).astype(np.int) + root_id_blocks = np.linspace(1, root_id_max, n_threads*3).astype(int) cg_info = self.cg.get_serialized_info() del (cg_info['credentials']) diff --git a/pychunkedgraph/meshing/meshgen.py b/pychunkedgraph/meshing/meshgen.py index ec858b3e5..b391333ff 100644 --- a/pychunkedgraph/meshing/meshgen.py +++ b/pychunkedgraph/meshing/meshgen.py @@ -73,10 +73,10 @@ def get_remapped_segmentation( cv = cloudvolume.CloudVolume(cg.cv.cloudpath, mip=mip) mip_diff = mip - cg.cv.mip - mip_chunk_size = cg.chunk_size.astype(np.int) / np.array( + mip_chunk_size = cg.chunk_size.astype(int) / np.array( [2 ** mip_diff, 2 ** mip_diff, 1] ) - mip_chunk_size = mip_chunk_size.astype(np.int) + mip_chunk_size = mip_chunk_size.astype(int) chunk_start = ( cg.cv.mip_voxel_offset(mip) @@ -147,10 +147,10 @@ def get_remapped_seg_for_lvl2_nodes( cv = cloudvolume.CloudVolume(cg.cv.cloudpath, mip=mip) mip_diff = mip - cg.cv.mip - mip_chunk_size = cg.chunk_size.astype(np.int) / np.array( + mip_chunk_size = cg.chunk_size.astype(int) / np.array( [2 ** mip_diff, 2 ** mip_diff, 1] ) - mip_chunk_size = mip_chunk_size.astype(np.int) + mip_chunk_size = mip_chunk_size.astype(int) chunk_start = ( cg.cv.mip_voxel_offset(mip) @@ -313,7 +313,7 @@ def _get_root_ids(args): root_ids = np.zeros(len(lx_ids), dtype=np.uint64) n_jobs = np.min([n_threads, len(lx_ids)]) multi_args = [] - start_ids = np.linspace(0, len(lx_ids), n_jobs + 1).astype(np.int) + start_ids = np.linspace(0, len(lx_ids), n_jobs + 1).astype(int) for i_block in range(n_jobs): multi_args.append([start_ids[i_block], start_ids[i_block + 1]]) @@ -480,7 +480,7 @@ def _get_root_ids(args): root_ids = np.zeros(len(combined_ids), dtype=np.uint64) n_jobs = np.min([n_threads, len(combined_ids)]) multi_args = [] - start_ids = np.linspace(0, len(combined_ids), n_jobs + 1).astype(np.int) + start_ids = np.linspace(0, len(combined_ids), n_jobs + 1).astype(int) for i_block in range(n_jobs): multi_args.append([start_ids[i_block], start_ids[i_block + 1]]) diff --git a/pychunkedgraph/meshing/meshgen_utils.py b/pychunkedgraph/meshing/meshgen_utils.py index 363786823..b56ce18c6 100644 --- a/pychunkedgraph/meshing/meshgen_utils.py +++ b/pychunkedgraph/meshing/meshgen_utils.py @@ -69,7 +69,7 @@ def get_mesh_block_shape_for_mip(cg, graphlayer: int, graphlayer_chunksize = cg.chunk_size * cg.fan_out ** np.max([0, graphlayer - 2]) - return np.floor_divide(graphlayer_chunksize, distortion, dtype=np.int, + return np.floor_divide(graphlayer_chunksize, distortion, dtype=int, casting='unsafe') diff --git a/pychunkedgraph/rechunking/transformer.py b/pychunkedgraph/rechunking/transformer.py index 5f4ed3707..38bf51ac6 100644 --- a/pychunkedgraph/rechunking/transformer.py +++ b/pychunkedgraph/rechunking/transformer.py @@ -44,9 +44,9 @@ def rewrite_single_segmentation_block(file_path, from_cv=None, to_cv=None, dx, dy, dz, _ = os.path.basename(file_path).split("_") - x_start, x_end = np.array(dx.split("-"), dtype=np.int) - y_start, y_end = np.array(dy.split("-"), dtype=np.int) - z_start, z_end = np.array(dz.split("-"), dtype=np.int) + x_start, x_end = np.array(dx.split("-"), dtype=int) + y_start, y_end = np.array(dy.split("-"), dtype=int) + z_start, z_end = np.array(dz.split("-"), dtype=int) bbox = to_cv.bounds.to_list()[3:] if x_end > bbox[0]: diff --git a/pychunkedgraph/tests/helpers.py b/pychunkedgraph/tests/helpers.py index 8097d9c77..77d652004 100644 --- a/pychunkedgraph/tests/helpers.py +++ b/pychunkedgraph/tests/helpers.py @@ -31,7 +31,7 @@ def to_list(self): class CloudVolumeMock(object): def __init__(self): - self.resolution = np.array([1, 1, 1], dtype=np.int) + self.resolution = np.array([1, 1, 1], dtype=int) self.bounds = CloudVolumeBounds() diff --git a/pychunkedgraph/tests/test_uncategorized.py b/pychunkedgraph/tests/test_uncategorized.py index e0d1b253d..d47c4ef84 100644 --- a/pychunkedgraph/tests/test_uncategorized.py +++ b/pychunkedgraph/tests/test_uncategorized.py @@ -775,7 +775,7 @@ def test_get_subgraph_edges(self, gen_graph_simplequerytest): def test_get_subgraph_nodes_bb(self, gen_graph_simplequerytest): cgraph = gen_graph_simplequerytest - bb = np.array([[1, 0, 0], [2, 1, 1]], dtype=np.int) + bb = np.array([[1, 0, 0], [2, 1, 1]], dtype=int) bb_coord = bb * cgraph.chunk_size childs_1 = cgraph.get_subgraph_nodes(cgraph.get_root(to_label(cgraph, 1, 1, 0, 0, 1)), bounding_box=bb) @@ -2554,7 +2554,7 @@ def test_cut_merge_history(self, gen_graph): mincut=False).new_root_ids assert len(split_roots) == 2 - timestamp_after_split = datetime.utcnow() + timestamp_after_split = datetime.utcnow() merge_roots = cgraph.add_edges("Jane Doe", [to_label(cgraph, 1, 0, 0, 0, 0), to_label(cgraph, 1, 1, 0, 0, 0)], @@ -2589,7 +2589,7 @@ def test_cut_merge_history(self, gen_graph): assert(new_roots2[0]==merge_root) assert(len(old_roots2)==2) assert(np.all(np.isin(old_roots2, split_roots))) - + new_roots3, old_roots3 = cgraph.get_delta_roots(timestamp_before_split, timestamp_after_merge) assert(len(new_roots3)==1) From 1db54d61405400b2f88b8b8d9efb5405e1fcc671 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 21:52:00 +0000 Subject: [PATCH 11/12] add init file to tests --- pychunkedgraph/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pychunkedgraph/tests/__init__.py diff --git a/pychunkedgraph/tests/__init__.py b/pychunkedgraph/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From 24a1b2747d2830f14ab9849356addeb0713cd283 Mon Sep 17 00:00:00 2001 From: Akhilesh Halageri Date: Tue, 25 Mar 2025 23:54:25 +0000 Subject: [PATCH 12/12] fix import --- pychunkedgraph/tests/test_uncategorized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pychunkedgraph/tests/test_uncategorized.py b/pychunkedgraph/tests/test_uncategorized.py index d47c4ef84..59a3743d8 100644 --- a/pychunkedgraph/tests/test_uncategorized.py +++ b/pychunkedgraph/tests/test_uncategorized.py @@ -16,7 +16,7 @@ from google.cloud import bigtable from grpc._channel import _Rendezvous -from helpers import (bigtable_emulator, create_chunk, gen_graph, +from .helpers import (bigtable_emulator, create_chunk, gen_graph, gen_graph_simplequerytest, lock_expired_timedelta_override, to_label) from pychunkedgraph.backend import chunkedgraph