diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7f17901d1..a4992d600d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ ci: - biome-check # Uses local npm/npx, not available in pre-commit CI repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace exclude_types: @@ -37,7 +37,7 @@ repos: - id: no-commit-to-branch - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.15.8 hooks: - id: ruff args: [--fix] @@ -56,7 +56,7 @@ repos: # language: python - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.21.2 hooks: - id: pyupgrade entry: pyupgrade --py39-plus --keep-runtime-typing @@ -87,7 +87,7 @@ repos: types_or: [javascript, jsx, ts, tsx, json] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.42.0 + rev: v0.48.0 hooks: - id: markdownlint args: [--disable, MD013] @@ -95,7 +95,7 @@ repos: args: [--disable, MD013] - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.2 + rev: v8.30.0 hooks: - id: gitleaks @@ -105,7 +105,7 @@ repos: - id: htmlhint - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta + rev: v2.14.0 hooks: - id: hadolint-docker args: diff --git a/backend/account_v2/custom_auth_middleware.py b/backend/account_v2/custom_auth_middleware.py index 59c3e85a5b..0589fa8961 100644 --- a/backend/account_v2/custom_auth_middleware.py +++ b/backend/account_v2/custom_auth_middleware.py @@ -1,6 +1,8 @@ import logging import uuid +from backend.constants import RequestHeader, RequestMethod +from backend.internal_api_constants import INTERNAL_API_PREFIX from django.conf import settings from django.http import HttpRequest, HttpResponse, JsonResponse from utils.constants import Account @@ -10,8 +12,6 @@ from account_v2.authentication_plugin_registry import AuthenticationPluginRegistry from account_v2.authentication_service import AuthenticationService from account_v2.constants import Common -from backend.constants import RequestHeader, RequestMethod -from backend.internal_api_constants import INTERNAL_API_PREFIX logger = logging.getLogger(__name__) diff --git a/backend/account_v2/models.py b/backend/account_v2/models.py index 5a3250bdd8..e76b7362c6 100644 --- a/backend/account_v2/models.py +++ b/backend/account_v2/models.py @@ -1,10 +1,9 @@ import uuid +from backend.constants import FieldLengthConstants as FieldLength from django.contrib.auth.models import AbstractUser, Group, Permission from django.db import models -from backend.constants import FieldLengthConstants as FieldLength - NAME_SIZE = 64 KEY_SIZE = 64 diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.py index a30b106a8a..5102199a45 100644 --- a/backend/adapter_processor_v2/adapter_processor.py +++ b/backend/adapter_processor_v2/adapter_processor.py @@ -8,6 +8,13 @@ from django.core.exceptions import ObjectDoesNotExist from platform_settings_v2.platform_auth_service import PlatformAuthenticationService from tenant_account_v2.organization_member_service import OrganizationMemberService +from unstract.sdk1.adapters.adapterkit import Adapterkit +from unstract.sdk1.adapters.base import Adapter +from unstract.sdk1.adapters.x2text.constants import X2TextConstants +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.embedding import EmbeddingCompat +from unstract.sdk1.exceptions import SdkError +from unstract.sdk1.llm import LLM from adapter_processor_v2.constants import AdapterKeys, AllowedDomains from adapter_processor_v2.exceptions import ( @@ -16,13 +23,6 @@ InValidAdapterId, TestAdapterError, ) -from unstract.sdk1.adapters.adapterkit import Adapterkit -from unstract.sdk1.adapters.base import Adapter -from unstract.sdk1.adapters.x2text.constants import X2TextConstants -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.embedding import EmbeddingCompat -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.llm import LLM from .models import AdapterInstance, UserDefaultAdapter diff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.py index 9356b19b7c..f6dc868e6d 100644 --- a/backend/adapter_processor_v2/exceptions.py +++ b/backend/adapter_processor_v2/exceptions.py @@ -1,7 +1,7 @@ from rest_framework.exceptions import APIException +from unstract.sdk1.exceptions import SdkError from adapter_processor_v2.constants import AdapterKeys -from unstract.sdk1.exceptions import SdkError class IdIsMandatory(APIException): diff --git a/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py b/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py index 304fc835a3..eaf742a538 100644 --- a/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py +++ b/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py @@ -88,7 +88,7 @@ def _list_deprecated(self) -> None: ) for adapter in deprecated_adapters: - self.stdout.write(f"\n{'='*80}") + self.stdout.write(f"\n{'=' * 80}") self.stdout.write(f"UUID: {adapter.id}") self.stdout.write(f"Name: {adapter.adapter_name}") self.stdout.write(f"Type: {adapter.adapter_type}") @@ -258,7 +258,7 @@ def _generate_report(self) -> None: self.stdout.write(f"Total adapters: {total_adapters}") self.stdout.write(f"Deprecated adapters: {deprecated_count}") self.stdout.write( - f"Deprecation rate: {(deprecated_count/total_adapters*100):.1f}%\n" + f"Deprecation rate: {(deprecated_count / total_adapters * 100):.1f}%\n" if total_adapters > 0 else "N/A\n" ) diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.py index 13be07cdc0..f391878c8a 100644 --- a/backend/adapter_processor_v2/models.py +++ b/backend/adapter_processor_v2/models.py @@ -9,6 +9,9 @@ from django.db import models from django.db.models import QuerySet from tenant_account_v2.models import OrganizationMember +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.exceptions import SdkError +from unstract.sdk1.llm import LLM from utils.exceptions import InvalidEncryptionKey from utils.models.base_model import BaseModel from utils.models.organization_mixin import ( @@ -16,10 +19,6 @@ DefaultOrganizationMixin, ) -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.llm import LLM - logger = logging.getLogger(__name__) ADAPTER_NAME_SIZE = 128 diff --git a/backend/adapter_processor_v2/serializers.py b/backend/adapter_processor_v2/serializers.py index 627731ed84..4f95b0dfbb 100644 --- a/backend/adapter_processor_v2/serializers.py +++ b/backend/adapter_processor_v2/serializers.py @@ -2,18 +2,18 @@ from typing import Any from account_v2.serializer import UserSerializer +from backend.constants import FieldLengthConstants as FLC +from backend.serializers import AuditSerializer from cryptography.fernet import Fernet from django.conf import settings from rest_framework import serializers from rest_framework.serializers import ModelSerializer +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.constants import Common as common from utils.input_sanitizer import validate_name_field, validate_no_html_tags from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.constants import AdapterKeys -from backend.constants import FieldLengthConstants as FLC -from backend.serializers import AuditSerializer -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.constants import Common as common from .models import AdapterInstance, UserDefaultAdapter diff --git a/backend/api_v2/management/commands/delete_org_rate_limit.py b/backend/api_v2/management/commands/delete_org_rate_limit.py index dbd9de859b..bdec54804c 100644 --- a/backend/api_v2/management/commands/delete_org_rate_limit.py +++ b/backend/api_v2/management/commands/delete_org_rate_limit.py @@ -86,7 +86,7 @@ def handle(self, *args, **options): try: usage = APIDeploymentRateLimiter.get_current_usage(organization) self.stdout.write( - f'\nCurrent usage: {usage["org_count"]}/{default_limit} concurrent requests' + f"\nCurrent usage: {usage['org_count']}/{default_limit} concurrent requests" ) if usage["org_count"] >= default_limit: diff --git a/backend/api_v2/management/commands/get_org_rate_limit.py b/backend/api_v2/management/commands/get_org_rate_limit.py index ab93073a30..183d8f8d98 100644 --- a/backend/api_v2/management/commands/get_org_rate_limit.py +++ b/backend/api_v2/management/commands/get_org_rate_limit.py @@ -65,10 +65,10 @@ def handle(self, *args, **options): self.stdout.write("\n--- Current Usage ---") self.stdout.write( - f'Organization: {usage["org_count"]}/{usage["org_limit"]} concurrent requests' + f"Organization: {usage['org_count']}/{usage['org_limit']} concurrent requests" ) self.stdout.write( - f'Global System: {usage["global_count"]}/{usage["global_limit"]} concurrent requests' + f"Global System: {usage['global_count']}/{usage['global_limit']} concurrent requests" ) # Usage percentage diff --git a/backend/api_v2/management/commands/list_org_rate_limits.py b/backend/api_v2/management/commands/list_org_rate_limits.py index 7c2d9284b6..3a9b543e8b 100644 --- a/backend/api_v2/management/commands/list_org_rate_limits.py +++ b/backend/api_v2/management/commands/list_org_rate_limits.py @@ -37,7 +37,7 @@ def handle(self, *args, **options): usage = APIDeploymentRateLimiter.get_current_usage(org) pct = (usage["org_count"] / limit * 100) if limit > 0 else 0 self.stdout.write( - f' Usage: {usage["org_count"]}/{limit} ({pct:.1f}%)' + f" Usage: {usage['org_count']}/{limit} ({pct:.1f}%)" ) except Exception as e: self.stdout.write(f" Usage: Error - {e}") diff --git a/backend/api_v2/management/commands/set_org_rate_limit.py b/backend/api_v2/management/commands/set_org_rate_limit.py index 6652ea775b..59aeeecc3b 100644 --- a/backend/api_v2/management/commands/set_org_rate_limit.py +++ b/backend/api_v2/management/commands/set_org_rate_limit.py @@ -55,7 +55,7 @@ def handle(self, *args, **options): usage = APIDeploymentRateLimiter.get_current_usage(organization) self.stdout.write( self.style.WARNING( - f'Current usage: {usage["org_count"]}/{limit} concurrent requests' + f"Current usage: {usage['org_count']}/{limit} concurrent requests" ) ) diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index 7c9a5a7696..fb1514872a 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -4,6 +4,7 @@ from typing import Any from urllib.parse import urlparse +from backend.serializers import AuditSerializer from django.apps import apps from django.core.validators import RegexValidator from pipeline_v2.models import Pipeline @@ -30,7 +31,6 @@ from api_v2.constants import ApiExecution from api_v2.models import APIDeployment, APIKey -from backend.serializers import AuditSerializer class APIDeploymentSerializer(IntegrityErrorMixin, AuditSerializer): diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py index caf4df575b..c7ca940752 100644 --- a/backend/backend/exceptions.py +++ b/backend/backend/exceptions.py @@ -3,7 +3,6 @@ from rest_framework.exceptions import APIException from rest_framework.response import Response from rest_framework.views import exception_handler - from unstract.connectors.exceptions import ConnectorBaseException, ConnectorError diff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.py index 2d9eef31da..022a00daef 100644 --- a/backend/configuration/config_registry.py +++ b/backend/configuration/config_registry.py @@ -165,9 +165,9 @@ def cast_value(cls, key_name: str, raw_value: Any) -> Any: converters = { ConfigType.INT: int, - ConfigType.BOOL: lambda v: v.lower() in ("true", "1") - if isinstance(v, str) - else bool(v), + ConfigType.BOOL: lambda v: ( + v.lower() in ("true", "1") if isinstance(v, str) else bool(v) + ), ConfigType.JSON: lambda v: __import__("json").loads(v), ConfigType.STRING: str, } diff --git a/backend/configuration/enums.py b/backend/configuration/enums.py index f35033a9f7..dd27fb289c 100644 --- a/backend/configuration/enums.py +++ b/backend/configuration/enums.py @@ -64,9 +64,9 @@ class ConfigKey(Enum): def cast_value(self, raw_value: Any): converters = { ConfigType.INT: int, - ConfigType.BOOL: lambda v: v.lower() in ("true", "1") - if isinstance(v, str) - else bool(v), + ConfigType.BOOL: lambda v: ( + v.lower() in ("true", "1") if isinstance(v, str) else bool(v) + ), ConfigType.JSON: json.loads, ConfigType.STRING: str, } diff --git a/backend/connector_auth_v2/pipeline/google.py b/backend/connector_auth_v2/pipeline/google.py index d585bd0857..9b6bec3ec3 100644 --- a/backend/connector_auth_v2/pipeline/google.py +++ b/backend/connector_auth_v2/pipeline/google.py @@ -3,7 +3,6 @@ from connector_auth_v2.constants import SocialAuthConstants as AuthConstants from connector_auth_v2.exceptions import EnrichConnectorMetadataException from connector_processor.constants import ConnectorKeys - from unstract.connectors.filesystems.google_drive.constants import GDriveConstants diff --git a/backend/connector_processor/connector_processor.py b/backend/connector_processor/connector_processor.py index 332aaaeb48..9f0ecaa247 100644 --- a/backend/connector_processor/connector_processor.py +++ b/backend/connector_processor/connector_processor.py @@ -3,11 +3,16 @@ import logging from typing import Any +from backend.exceptions import UnstractFSException from connector_auth_v2.constants import ConnectorAuthKey from connector_auth_v2.pipeline.common import ConnectorAuthHelper from connector_v2.constants import ConnectorInstanceKey as CIKey +from unstract.connectors.base import UnstractConnector +from unstract.connectors.connectorkit import Connectorkit +from unstract.connectors.enums import ConnectorMode +from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError +from unstract.connectors.filesystems.ucs import UnstractCloudStorage -from backend.exceptions import UnstractFSException from connector_processor.constants import ConnectorKeys from connector_processor.exceptions import ( InvalidConnectorID, @@ -15,11 +20,6 @@ OAuthTimeOut, TestConnectorInputError, ) -from unstract.connectors.base import UnstractConnector -from unstract.connectors.connectorkit import Connectorkit -from unstract.connectors.enums import ConnectorMode -from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError -from unstract.connectors.filesystems.ucs import UnstractCloudStorage logger = logging.getLogger(__name__) @@ -145,8 +145,7 @@ def test_connectors(connector_id: str, credentials: dict[str, Any]) -> bool: ) except Exception as exc: logger.error( - "Error while testing file based OAuth supported " - "connectors: %s", + "Error while testing file based OAuth supported connectors: %s", exc, ) raise OAuthTimeOut() diff --git a/backend/connector_processor/exceptions.py b/backend/connector_processor/exceptions.py index 2deb364699..2b9cbc0b00 100644 --- a/backend/connector_processor/exceptions.py +++ b/backend/connector_processor/exceptions.py @@ -1,6 +1,5 @@ -from rest_framework.exceptions import APIException - from backend.exceptions import UnstractBaseException +from rest_framework.exceptions import APIException from unstract.connectors.exceptions import ConnectorError diff --git a/backend/connector_processor/serializers.py b/backend/connector_processor/serializers.py index bd0bcf0b8f..a2ca7d0075 100644 --- a/backend/connector_processor/serializers.py +++ b/backend/connector_processor/serializers.py @@ -1,6 +1,6 @@ +from backend.constants import FieldLengthConstants as FLC from rest_framework import serializers -from backend.constants import FieldLengthConstants as FLC from connector_processor.constants import ConnectorKeys diff --git a/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py index 73dd12f04b..9269767e47 100644 --- a/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py +++ b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py @@ -221,7 +221,7 @@ def _fix_remaining_duplicate_names(connector_instance_model: Any) -> int: for i in range(0, len(duplicates), batch_size): batch = duplicates[i : i + batch_size] logger.info( - f"Processing batch {i//batch_size + 1}/{(len(duplicates)-1)//batch_size + 1}" + f"Processing batch {i // batch_size + 1}/{(len(duplicates) - 1) // batch_size + 1}" ) for dup_info in batch: diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py index c3d0e6108b..47985ee924 100644 --- a/backend/connector_v2/models.py +++ b/backend/connector_v2/models.py @@ -3,6 +3,7 @@ from typing import Any from account_v2.models import User +from backend.constants import FieldLengthConstants as FLC from connector_auth_v2.models import ConnectorAuth from connector_processor.connector_processor import ConnectorProcessor from connector_processor.constants import ConnectorKeys @@ -14,8 +15,6 @@ DefaultOrganizationMixin, ) -from backend.constants import FieldLengthConstants as FLC - CONNECTOR_NAME_SIZE = 128 VERSION_NAME_SIZE = 64 diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.py index 5517bc5257..038c0253bf 100644 --- a/backend/connector_v2/serializers.py +++ b/backend/connector_v2/serializers.py @@ -2,18 +2,18 @@ from collections import OrderedDict from typing import Any +from backend.serializers import AuditSerializer from connector_auth_v2.models import ConnectorAuth from connector_auth_v2.pipeline.common import ConnectorAuthHelper from connector_processor.connector_processor import ConnectorProcessor from connector_processor.constants import ConnectorKeys from connector_processor.exceptions import OAuthTimeOut from rest_framework.serializers import CharField, SerializerMethodField +from unstract.connectors.filesystems.ucs import UnstractCloudStorage from utils.fields import EncryptedBinaryFieldSerializer from utils.input_sanitizer import validate_name_field -from backend.serializers import AuditSerializer from connector_v2.constants import ConnectorInstanceKey as CIKey -from unstract.connectors.filesystems.ucs import UnstractCloudStorage from .models import ConnectorInstance diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py index 09a33be983..02aaa68910 100644 --- a/backend/connector_v2/views.py +++ b/backend/connector_v2/views.py @@ -2,6 +2,7 @@ from typing import Any from account_v2.custom_exceptions import DuplicateData +from backend.constants import RequestKey from connector_auth_v2.constants import ConnectorAuthKey from connector_auth_v2.exceptions import CacheMissException, MissingParamException from connector_auth_v2.pipeline.common import ConnectorAuthHelper @@ -14,12 +15,11 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning +from unstract.connectors.connectorkit import Connectorkit +from unstract.connectors.enums import ConnectorMode from utils.filtering import FilterHelper -from backend.constants import RequestKey from connector_v2.constants import ConnectorInstanceKey as CIKey -from unstract.connectors.connectorkit import Connectorkit -from unstract.connectors.enums import ConnectorMode from .exceptions import DeleteConnectorInUseError from .models import ConnectorInstance diff --git a/backend/dashboard_metrics/services.py b/backend/dashboard_metrics/services.py index e382f62617..4f77573d2e 100644 --- a/backend/dashboard_metrics/services.py +++ b/backend/dashboard_metrics/services.py @@ -17,6 +17,7 @@ from django.db.models import CharField, Count, OuterRef, Subquery, Sum from django.db.models.functions import Cast, Coalesce, TruncDay, TruncHour, TruncWeek from pipeline_v2.models import Pipeline +from unstract.core.data_models import ExecutionStatus from usage_v2.models import Usage from workflow_manager.execution.enum import ExecutionEntity from workflow_manager.file_execution.models import WorkflowFileExecution @@ -24,7 +25,6 @@ from workflow_manager.workflow_v2.models.workflow import Workflow from dashboard_metrics.models import Granularity -from unstract.core.data_models import ExecutionStatus def _get_hitl_queue_model(): diff --git a/backend/file_management/exceptions.py b/backend/file_management/exceptions.py index 75f45ed30a..b7254be9f0 100644 --- a/backend/file_management/exceptions.py +++ b/backend/file_management/exceptions.py @@ -1,6 +1,5 @@ -from rest_framework.exceptions import APIException - from backend.exceptions import UnstractBaseException +from rest_framework.exceptions import APIException class MissingConnectorParams(APIException): diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py index ab56c4dba5..568f6add8a 100644 --- a/backend/file_management/file_management_helper.py +++ b/backend/file_management/file_management_helper.py @@ -12,6 +12,8 @@ from django.http import StreamingHttpResponse from fsspec import AbstractFileSystem from pydrive2.files import ApiRequestError +from unstract.connectors.filesystems import connectors as fs_connectors +from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from file_management.exceptions import ( ConnectorApiRequestError, @@ -25,8 +27,6 @@ TenantDirCreationError, ) from file_management.file_management_dto import FileInformation -from unstract.connectors.filesystems import connectors as fs_connectors -from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem logger = logging.getLogger(__name__) diff --git a/backend/file_management/views.py b/backend/file_management/views.py index b2875d4251..71326f60b5 100644 --- a/backend/file_management/views.py +++ b/backend/file_management/views.py @@ -9,6 +9,8 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning +from unstract.connectors.exceptions import ConnectorError +from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS from utils.user_session import UserSessionUtils from file_management.exceptions import ( @@ -23,8 +25,6 @@ FileListRequestSerializer, FileUploadSerializer, ) -from unstract.connectors.exceptions import ConnectorError -from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS logger = logging.getLogger(__name__) diff --git a/backend/middleware/exception.py b/backend/middleware/exception.py index f5ea1a6f7c..ffcc210551 100644 --- a/backend/middleware/exception.py +++ b/backend/middleware/exception.py @@ -8,7 +8,6 @@ from drf_standardized_errors.handler import exception_handler from rest_framework.request import Request from rest_framework.response import Response - from unstract.sdk1.exceptions import SdkError logger = logging.getLogger(__name__) diff --git a/backend/migrating/v2/management/commands/migrate_to_v2.py b/backend/migrating/v2/management/commands/migrate_to_v2.py index 1b8d5a8bc7..513923cade 100644 --- a/backend/migrating/v2/management/commands/migrate_to_v2.py +++ b/backend/migrating/v2/management/commands/migrate_to_v2.py @@ -367,7 +367,7 @@ def _bump_auto_increment_id( if max_id is None: logger.info( - f"Table '{dest_table}' is empty. No need to adjust " "auto-increment." + f"Table '{dest_table}' is empty. No need to adjust auto-increment." ) return diff --git a/backend/notification_v2/provider/webhook/webhook.py b/backend/notification_v2/provider/webhook/webhook.py index 549336a5f9..c395ef72ec 100644 --- a/backend/notification_v2/provider/webhook/webhook.py +++ b/backend/notification_v2/provider/webhook/webhook.py @@ -1,6 +1,7 @@ import logging from backend.celery_service import app as celery_app + from notification_v2.enums import AuthorizationType from notification_v2.provider.notification_provider import NotificationProvider @@ -67,8 +68,7 @@ def get_headers(self): ) except ValueError: raise ValueError( - "Unsupported authorization type: " - f"{self.notification.authorization_type}" + f"Unsupported authorization type: {self.notification.authorization_type}" ) authorization_key = self.notification.authorization_key authorization_header = self.notification.authorization_header diff --git a/backend/pipeline_v2/exceptions.py b/backend/pipeline_v2/exceptions.py index 8b87f661a5..011bf74db5 100644 --- a/backend/pipeline_v2/exceptions.py +++ b/backend/pipeline_v2/exceptions.py @@ -28,7 +28,7 @@ def __init__( ): if pipeline_name: self.default_detail = ( - f"Pipeline '{pipeline_name}' is inactive, " "please activate the pipeline" + f"Pipeline '{pipeline_name}' is inactive, please activate the pipeline" ) super().__init__(detail, code) diff --git a/backend/pipeline_v2/manager.py b/backend/pipeline_v2/manager.py index 9be9d80317..e979b74a26 100644 --- a/backend/pipeline_v2/manager.py +++ b/backend/pipeline_v2/manager.py @@ -1,6 +1,7 @@ import logging from typing import Any +from backend.constants import RequestHeader from django.conf import settings from django.urls import reverse from rest_framework.request import Request @@ -9,7 +10,6 @@ from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey from workflow_manager.workflow_v2.views import WorkflowViewSet -from backend.constants import RequestHeader from pipeline_v2.constants import PipelineKey, PipelineURL from pipeline_v2.models import Pipeline from pipeline_v2.pipeline_processor import PipelineProcessor diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py index 00b12c8483..544c343e96 100644 --- a/backend/pipeline_v2/models.py +++ b/backend/pipeline_v2/models.py @@ -1,6 +1,7 @@ import uuid from account_v2.models import User +from backend.constants import FieldLengthConstants as FieldLength from django.conf import settings from django.db import models from django.db.models import Q @@ -12,8 +13,6 @@ from utils.user_context import UserContext from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.constants import FieldLengthConstants as FieldLength - APP_ID_LENGTH = 32 PIPELINE_NAME_LENGTH = 32 diff --git a/backend/pipeline_v2/piepline_api_execution_views.py b/backend/pipeline_v2/piepline_api_execution_views.py index a2b88a13df..02a98be003 100644 --- a/backend/pipeline_v2/piepline_api_execution_views.py +++ b/backend/pipeline_v2/piepline_api_execution_views.py @@ -1,11 +1,11 @@ import logging from typing import Any +from backend.celery_service import app as celery_app from rest_framework import status, views from rest_framework.request import Request from rest_framework.response import Response -from backend.celery_service import app as celery_app from pipeline_v2.deployment_helper import DeploymentHelper from pipeline_v2.models import Pipeline diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py index f887042b9a..203fd37e2b 100644 --- a/backend/pipeline_v2/serializers/crud.py +++ b/backend/pipeline_v2/serializers/crud.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Any +from backend.serializers import AuditSerializer from croniter import croniter from django.conf import settings from django.utils import timezone @@ -13,14 +14,12 @@ from rest_framework import serializers from rest_framework.serializers import SerializerMethodField from scheduler.helper import SchedulerHelper +from unstract.connectors.connectorkit import Connectorkit from utils.serializer.integrity_error_mixin import IntegrityErrorMixin from utils.serializer_utils import SerializerUtils from workflow_manager.endpoint_v2.models import WorkflowEndpoint from workflow_manager.workflow_v2.models.execution import WorkflowExecution -from backend.serializers import AuditSerializer -from unstract.connectors.connectorkit import Connectorkit - logger = logging.getLogger(__name__) DEPLOYMENT_ENDPOINT = settings.API_DEPLOYMENT_PATH_PREFIX + "/pipeline" diff --git a/backend/platform_api/serializers.py b/backend/platform_api/serializers.py index 132e4feaa0..8739c61125 100644 --- a/backend/platform_api/serializers.py +++ b/backend/platform_api/serializers.py @@ -1,9 +1,9 @@ import re +from backend.serializers import AuditSerializer from rest_framework import serializers from utils.user_context import UserContext -from backend.serializers import AuditSerializer from platform_api.models import PlatformApiKey # Alphanumeric, spaces, hyphens, underscores, periods, commas, colons, diff --git a/backend/platform_settings_v2/serializers.py b/backend/platform_settings_v2/serializers.py index 24bd93ec5c..3595020213 100644 --- a/backend/platform_settings_v2/serializers.py +++ b/backend/platform_settings_v2/serializers.py @@ -1,7 +1,6 @@ from account_v2.models import PlatformKey -from rest_framework import serializers - from backend.serializers import AuditSerializer +from rest_framework import serializers class PlatformKeySerializer(AuditSerializer): diff --git a/backend/prompt_studio/prompt_profile_manager_v2/serializers.py b/backend/prompt_studio/prompt_profile_manager_v2/serializers.py index b6c2e13431..16df3e7b21 100644 --- a/backend/prompt_studio/prompt_profile_manager_v2/serializers.py +++ b/backend/prompt_studio/prompt_profile_manager_v2/serializers.py @@ -1,8 +1,8 @@ import logging from adapter_processor_v2.adapter_processor import AdapterProcessor - from backend.serializers import AuditSerializer + from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerKeys from .models import ProfileManager diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py index 406c157efc..13d6ca26c7 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/models.py +++ b/backend/prompt_studio/prompt_studio_core_v2/models.py @@ -6,6 +6,8 @@ from adapter_processor_v2.models import AdapterInstance from django.db import models from django.db.models import QuerySet +from unstract.sdk1.file_storage.constants import StorageType +from unstract.sdk1.file_storage.env_helper import EnvHelper from utils.file_storage.constants import FileStorageKeys from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.models.base_model import BaseModel @@ -15,8 +17,6 @@ ) from prompt_studio.prompt_studio_core_v2.constants import DefaultPrompts -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper logger = logging.getLogger(__name__) diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py index 536bb16eb1..c1e0c46aab 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py +++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py @@ -1,13 +1,13 @@ import os from platform_settings_v2.platform_auth_service import PlatformAuthenticationService -from utils.file_storage.constants import FileStorageKeys - -from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys from unstract.sdk1.constants import LogLevel from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper from unstract.sdk1.tool.stream import StreamMixin +from utils.file_storage.constants import FileStorageKeys + +from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys class PromptIdeBaseTool(StreamMixin): diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py index b3dcbc95d1..4be71eb5fc 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py +++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py @@ -15,6 +15,14 @@ from django.db.models.manager import BaseManager from plugins import get_plugin from rest_framework.request import Request +from unstract.core.pubsub_helper import LogPublisher +from unstract.sdk1.constants import LogLevel +from unstract.sdk1.exceptions import IndexingError, SdkError +from unstract.sdk1.file_storage.constants import StorageType +from unstract.sdk1.file_storage.env_helper import EnvHelper +from unstract.sdk1.prompt import PromptTool +from unstract.sdk1.utils.indexing import IndexingUtils +from unstract.sdk1.utils.tool import ToolUtils from utils.file_storage.constants import FileStorageKeys from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.local_context import StateStore @@ -65,14 +73,6 @@ OutputManagerHelper, ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt -from unstract.core.pubsub_helper import LogPublisher -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.exceptions import IndexingError, SdkError -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper -from unstract.sdk1.prompt import PromptTool -from unstract.sdk1.utils.indexing import IndexingUtils -from unstract.sdk1.utils.tool import ToolUtils logger = logging.getLogger(__name__) diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py index 14ce69be5a..6bd1819b8e 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py @@ -4,14 +4,15 @@ from account_v2.models import User from account_v2.serializer import UserSerializer from adapter_processor_v2.models import AdapterInstance +from backend.serializers import AuditSerializer from django.core.exceptions import ObjectDoesNotExist from rest_framework import serializers from rest_framework.exceptions import ValidationError +from unstract.sdk1.adapters.enums import AdapterTypes from utils.FileValidator import FileValidator from utils.input_sanitizer import validate_name_field, validate_no_html_tags from utils.serializer.integrity_error_mixin import IntegrityErrorMixin -from backend.serializers import AuditSerializer from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys as TSKeys from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError @@ -20,7 +21,6 @@ ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer -from unstract.sdk1.adapters.enums import AdapterTypes from .models import CustomTool diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py index 014f825d17..6b919cc88b 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/views.py +++ b/backend/prompt_studio/prompt_studio_core_v2/views.py @@ -21,6 +21,7 @@ from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning from tool_instance_v2.models import ToolInstance +from unstract.sdk1.utils.common import Utils as CommonUtils from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.hubspot_notify import notify_hubspot_event from utils.user_context import UserContext @@ -70,7 +71,6 @@ from prompt_studio.prompt_studio_v2.constants import ToolStudioPromptErrors from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer -from unstract.sdk1.utils.common import Utils as CommonUtils from .models import CustomTool from .serializers import ( @@ -901,7 +901,7 @@ def sync_prompts(self, request: Request, pk: Any = None) -> Response: sync_result = PromptStudioHelper.sync_prompts(tool, import_data, request.user) response_data.update(sync_result) response_data["message"] = ( - f"Synced {sync_result['prompts_created']} prompts " f"into '{tool.tool_name}'" + f"Synced {sync_result['prompts_created']} prompts into '{tool.tool_name}'" ) return Response(response_data, status=status.HTTP_200_OK) diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py index 60ee406304..294c53a49b 100644 --- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py +++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py @@ -6,14 +6,14 @@ from django.db import models from django.db.models.signals import pre_delete from django.dispatch import receiver +from unstract.sdk1.constants import LogLevel +from unstract.sdk1.vector_db import VectorDB from utils.models.base_model import BaseModel from utils.user_context import UserContext from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool import PromptIdeBaseTool from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.vector_db import VectorDB logger = logging.getLogger(__name__) diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py index 275e4a0956..977164bb0d 100644 --- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py @@ -1,9 +1,8 @@ import json import logging -from usage_v2.helper import UsageHelper - from backend.serializers import AuditSerializer +from usage_v2.helper import UsageHelper from .models import PromptStudioOutputManager from .output_manager_util import OutputManagerUtils diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py index 6ce1f72095..5c79f9198d 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py @@ -6,6 +6,7 @@ from django.conf import settings from django.db import IntegrityError from plugins import get_plugin +from unstract.tool_registry.dto import Properties, Spec, Tool from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from prompt_studio.prompt_studio_core_v2.models import CustomTool @@ -14,7 +15,6 @@ PromptStudioOutputManager, ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt -from unstract.tool_registry.dto import Properties, Spec, Tool from .constants import JsonSchemaKey, PromptStudioRegistryKeys from .exceptions import ( diff --git a/backend/prompt_studio/prompt_studio_registry_v2/serializers.py b/backend/prompt_studio/prompt_studio_registry_v2/serializers.py index 838ec7d4db..403a85818b 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/serializers.py @@ -1,9 +1,8 @@ from typing import Any from account_v2.serializer import UserSerializer -from rest_framework import serializers - from backend.serializers import AuditSerializer +from rest_framework import serializers from .models import PromptStudioRegistry diff --git a/backend/prompt_studio/prompt_studio_v2/serializers.py b/backend/prompt_studio/prompt_studio_v2/serializers.py index e1adddc33c..d53fc9e9f2 100644 --- a/backend/prompt_studio/prompt_studio_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_v2/serializers.py @@ -1,6 +1,5 @@ -from rest_framework import serializers - from backend.serializers import AuditSerializer +from rest_framework import serializers from .models import ToolStudioPrompt diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.py index 2cb0d61502..d2b7abab22 100644 --- a/backend/scheduler/serializer.py +++ b/backend/scheduler/serializer.py @@ -1,10 +1,10 @@ import logging from typing import Any +from backend.constants import FieldLengthConstants as FieldLength from pipeline_v2.manager import PipelineManager from rest_framework import serializers -from backend.constants import FieldLengthConstants as FieldLength from scheduler.constants import SchedulerConstants as SC logger = logging.getLogger(__name__) diff --git a/backend/tenant_account_v2/models.py b/backend/tenant_account_v2/models.py index e5fdacda66..5bb8e99a6a 100644 --- a/backend/tenant_account_v2/models.py +++ b/backend/tenant_account_v2/models.py @@ -30,8 +30,7 @@ class OrganizationMember(DefaultOrganizationMixin): def __str__(self): # type: ignore return ( - f"OrganizationMember(" - f"{self.member_id}, role: {self.role}, user: {self.user})" + f"OrganizationMember({self.member_id}, role: {self.role}, user: {self.user})" ) objects = OrganizationMemberModelManager() diff --git a/backend/tenant_account_v2/serializer.py b/backend/tenant_account_v2/serializer.py index 7947464543..8f3f779ff4 100644 --- a/backend/tenant_account_v2/serializer.py +++ b/backend/tenant_account_v2/serializer.py @@ -92,7 +92,7 @@ class RemoveUserFromOrganizationSerializer(serializers.Serializer): ) def get_user_emails(self, validated_data: dict[str, list[str] | None]) -> list[str]: - return cast(list[str], validated_data.get(Common.USER_EMAILS, [])) + return cast("list[str]", validated_data.get(Common.USER_EMAILS, [])) class ChangeUserRoleRequestSerializer(serializers.Serializer): diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.py index 81eb6e0e60..2f87cec1e5 100644 --- a/backend/tool_instance_v2/serializers.py +++ b/backend/tool_instance_v2/serializers.py @@ -4,21 +4,21 @@ from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.models import AdapterInstance +from backend.serializers import AuditSerializer from prompt_studio.prompt_studio_registry_v2.constants import PromptStudioRegistryKeys from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationError +from unstract.sdk1.adapters.enums import AdapterTypes +from unstract.tool_registry.dto import Tool +from unstract.tool_registry.tool_utils import ToolUtils from workflow_manager.workflow_v2.constants import WorkflowKey from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.serializers import AuditSerializer from tool_instance_v2.constants import ToolInstanceKey as TIKey from tool_instance_v2.constants import ToolKey from tool_instance_v2.exceptions import ToolDoesNotExist from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_instance_helper import ToolInstanceHelper from tool_instance_v2.tool_processor import ToolProcessor -from unstract.sdk1.adapters.enums import AdapterTypes -from unstract.tool_registry.dto import Tool -from unstract.tool_registry.tool_utils import ToolUtils logger = logging.getLogger(__name__) diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.py index 08f425f745..a5ff88dca9 100644 --- a/backend/tool_instance_v2/tool_instance_helper.py +++ b/backend/tool_instance_v2/tool_instance_helper.py @@ -10,17 +10,17 @@ from django.core.exceptions import ValidationError as DjangoValidationError from jsonschema.exceptions import ValidationError as JSONValidationError from prompt_studio.prompt_studio_registry_v2.models import PromptStudioRegistry +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.tool.validator import DefaultsGeneratingValidator +from unstract.tool_registry.constants import AdapterPropertyKey +from unstract.tool_registry.dto import Spec, Tool +from unstract.tool_registry.tool_utils import ToolUtils from workflow_manager.workflow_v2.constants import WorkflowKey from tool_instance_v2.constants import JsonSchemaKey from tool_instance_v2.exceptions import ToolSettingValidationError from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_processor import ToolProcessor -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.tool.validator import DefaultsGeneratingValidator -from unstract.tool_registry.constants import AdapterPropertyKey -from unstract.tool_registry.dto import Spec, Tool -from unstract.tool_registry.tool_utils import ToolUtils # Import agentic registry if available (cloud-only feature) try: diff --git a/backend/tool_instance_v2/tool_processor.py b/backend/tool_instance_v2/tool_processor.py index 5e33cceec4..3c25643835 100644 --- a/backend/tool_instance_v2/tool_processor.py +++ b/backend/tool_instance_v2/tool_processor.py @@ -8,13 +8,13 @@ from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import ( PromptStudioRegistryHelper, ) - -from tool_instance_v2.exceptions import ToolDoesNotExist from unstract.sdk1.constants import AdapterTypes from unstract.tool_registry.dto import Spec, Tool from unstract.tool_registry.tool_registry import ToolRegistry from unstract.tool_registry.tool_utils import ToolUtils +from tool_instance_v2.exceptions import ToolDoesNotExist + # Import agentic registry if available (cloud-only feature) try: from pluggable_apps.agentic_studio_registry.registry_helper import ( diff --git a/backend/tool_instance_v2/views.py b/backend/tool_instance_v2/views.py index c8321b2f4c..e39a36aa07 100644 --- a/backend/tool_instance_v2/views.py +++ b/backend/tool_instance_v2/views.py @@ -3,6 +3,7 @@ from typing import Any from account_v2.custom_exceptions import DuplicateData +from backend.constants import RequestKey from django.db import IntegrityError from django.db.models.query import QuerySet from rest_framework import serializers, status, viewsets @@ -14,7 +15,6 @@ from utils.user_session import UserSessionUtils from workflow_manager.workflow_v2.constants import WorkflowKey -from backend.constants import RequestKey from tool_instance_v2.constants import ToolInstanceErrors, ToolKey from tool_instance_v2.constants import ToolInstanceKey as TIKey from tool_instance_v2.exceptions import FetchToolListFailed, ToolFunctionIsMandatory diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.py index 3c28779c56..19eec1eaae 100644 --- a/backend/usage_v2/internal_views.py +++ b/backend/usage_v2/internal_views.py @@ -6,7 +6,6 @@ from rest_framework import status from rest_framework.request import Request from rest_framework.views import APIView - from unstract.core.data_models import UsageResponseData from .helper import UsageHelper diff --git a/backend/utils/FileValidator.py b/backend/utils/FileValidator.py index c4c66b473d..a6242bbace 100644 --- a/backend/utils/FileValidator.py +++ b/backend/utils/FileValidator.py @@ -7,7 +7,6 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.template.defaultfilters import filesizeformat from django.utils.translation import gettext_lazy as _ - from unstract.sdk1.file_storage.constants import FileOperationParams @@ -37,8 +36,7 @@ class FileValidator: "Allowed extensions are: '%(allowed_extensions)s.'" ) mime_message = _( - "MIME type '%(mimetype)s' is not valid. " - "Allowed types are: %(allowed_mimetypes)s." + "MIME type '%(mimetype)s' is not valid. Allowed types are: %(allowed_mimetypes)s." ) min_size_message = _( "The current file %(size)s, which is too small. " diff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.py index ea6e4278bf..f8110076c3 100644 --- a/backend/utils/cache_service.py +++ b/backend/utils/cache_service.py @@ -6,7 +6,6 @@ from django.conf import settings from django.core.cache import cache from django_redis import get_redis_connection - from unstract.core.cache.redis_client import create_redis_client redis_cache = get_redis_connection("default") diff --git a/backend/utils/dto.py b/backend/utils/dto.py index af10f475fe..7c6217ee5f 100644 --- a/backend/utils/dto.py +++ b/backend/utils/dto.py @@ -6,7 +6,6 @@ from typing import Any from django.utils import timezone as dj_timezone - from unstract.core.constants import LogFieldName from unstract.workflow_execution.enums import LogType diff --git a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py index 23a05633fd..511931f8ca 100644 --- a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py +++ b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py @@ -6,12 +6,11 @@ from file_management.exceptions import InvalidFileType from file_management.file_management_helper import FileManagerHelper -from utils.file_storage.constants import FileStorageConstants, FileStorageKeys - from unstract.core.utilities import UnstractUtils from unstract.sdk1.file_storage import FileStorage from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper +from utils.file_storage.constants import FileStorageConstants, FileStorageKeys logger = logging.getLogger(__name__) diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py index f2ff725820..75a9edece1 100644 --- a/backend/utils/log_events.py +++ b/backend/utils/log_events.py @@ -6,10 +6,10 @@ import socketio from django.conf import settings from django.core.wsgi import WSGIHandler - from unstract.core.cache.redis_client import create_redis_client from unstract.core.data_models import LogDataDTO from unstract.core.log_utils import get_validated_log_data, store_execution_log + from utils.constants import ExecutionLogConstants logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py index 66fec358f7..92bf693d7f 100644 --- a/backend/workflow_manager/endpoint_v2/base_connector.py +++ b/backend/workflow_manager/endpoint_v2/base_connector.py @@ -2,12 +2,11 @@ from typing import Any from fsspec import AbstractFileSystem -from utils.constants import Common -from utils.user_context import UserContext - from unstract.connectors.filesystems import connectors from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler +from utils.constants import Common +from utils.user_context import UserContext class BaseConnector(ExecutionFileHandler): diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.py index a89594e186..2330b42fd8 100644 --- a/backend/workflow_manager/endpoint_v2/database_utils.py +++ b/backend/workflow_manager/endpoint_v2/database_utils.py @@ -3,17 +3,16 @@ import logging from typing import Any +from unstract.connectors.databases import connectors as db_connectors +from unstract.connectors.databases.exceptions import UnstractDBConnectorException +from unstract.connectors.databases.unstract_db import UnstractDB +from unstract.connectors.exceptions import ConnectorError from utils.constants import Common from workflow_manager.endpoint_v2.constants import TableColumns from workflow_manager.endpoint_v2.enums import FileProcessingStatus from workflow_manager.endpoint_v2.exceptions import UnstractDBException from workflow_manager.workflow_v2.enums import AgentName, ColumnModes -from unstract.connectors.databases import connectors as db_connectors -from unstract.connectors.databases.exceptions import UnstractDBConnectorException -from unstract.connectors.databases.unstract_db import UnstractDB -from unstract.connectors.exceptions import ConnectorError - logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py index 093c64b905..f9a7507d2b 100644 --- a/backend/workflow_manager/endpoint_v2/destination.py +++ b/backend/workflow_manager/endpoint_v2/destination.py @@ -5,9 +5,14 @@ import os from typing import Any +from backend.exceptions import UnstractFSException from connector_v2.models import ConnectorInstance from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil from rest_framework.exceptions import APIException +from unstract.connectors.exceptions import ConnectorError +from unstract.sdk1.constants import ToolExecKey +from unstract.sdk1.tool.mime_types import EXT_MIME_MAP +from unstract.workflow_execution.constants import ToolOutputType from usage_v2.helper import UsageHelper from utils.user_context import UserContext from workflow_manager.endpoint_v2.base_connector import BaseConnector @@ -34,12 +39,7 @@ from workflow_manager.workflow_v2.models.file_history import FileHistory from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.exceptions import UnstractFSException -from unstract.connectors.exceptions import ConnectorError from unstract.filesystem import FileStorageType, FileSystem -from unstract.sdk1.constants import ToolExecKey -from unstract.sdk1.tool.mime_types import EXT_MIME_MAP -from unstract.workflow_execution.constants import ToolOutputType logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py index 204518f9ae..538390de53 100644 --- a/backend/workflow_manager/endpoint_v2/queue_utils.py +++ b/backend/workflow_manager/endpoint_v2/queue_utils.py @@ -5,11 +5,10 @@ from typing import Any from django.conf import settings -from utils.constants import Common -from workflow_manager.endpoint_v2.exceptions import UnstractQueueException - from unstract.connectors.queues import connectors as queue_connectors from unstract.connectors.queues.unstract_queue import UnstractQueue +from utils.constants import Common +from workflow_manager.endpoint_v2.exceptions import UnstractQueueException logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py index 23b0d04f2c..7a515d8ba8 100644 --- a/backend/workflow_manager/endpoint_v2/source.py +++ b/backend/workflow_manager/endpoint_v2/source.py @@ -15,6 +15,9 @@ from connector_v2.models import ConnectorInstance from django.core.files.uploadedfile import UploadedFile from django.db.models import Q +from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem +from unstract.sdk1.file_storage import FileStorage +from unstract.workflow_execution.enums import LogLevel, LogStage, LogState from utils.user_context import UserContext from workflow_manager.endpoint_v2.base_connector import BaseConnector from workflow_manager.endpoint_v2.constants import ( @@ -45,10 +48,7 @@ from workflow_manager.workflow_v2.models.file_history import FileHistory from workflow_manager.workflow_v2.models.workflow import Workflow -from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from unstract.filesystem import FileStorageType, FileSystem -from unstract.sdk1.file_storage import FileStorage -from unstract.workflow_execution.enums import LogLevel, LogStage, LogState logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/internal_serializers.py b/backend/workflow_manager/internal_serializers.py index bed98c6853..5167e4728c 100644 --- a/backend/workflow_manager/internal_serializers.py +++ b/backend/workflow_manager/internal_serializers.py @@ -12,6 +12,7 @@ FileExecutionStatusUpdateRequest, WorkflowFileExecutionData, ) + from workflow_manager.file_execution.models import WorkflowFileExecution from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.models.execution import WorkflowExecution diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.py index 94ecd7f76e..254f2f378e 100644 --- a/backend/workflow_manager/internal_views.py +++ b/backend/workflow_manager/internal_views.py @@ -13,9 +13,6 @@ from rest_framework.response import Response from rest_framework.views import APIView from tool_instance_v2.models import ToolInstance -from utils.constants import Account -from utils.local_context import StateStore -from utils.organization_utils import filter_queryset_by_organization # Import new dataclasses for WorkflowDefinitionAPIView from unstract.core.data_models import ( @@ -25,6 +22,10 @@ WorkflowEndpointConfigData, WorkflowEndpointConfigResponseData, ) +from utils.constants import Account +from utils.local_context import StateStore +from utils.organization_utils import filter_queryset_by_organization + from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils from workflow_manager.endpoint_v2.models import WorkflowEndpoint from workflow_manager.file_execution.models import WorkflowFileExecution @@ -1306,9 +1307,9 @@ class WorkflowSourceFilesAPIView(APIView): def post(self, request, workflow_id): """Get source files for a workflow execution.""" try: + from unstract.workflow_execution.enums import LogStage from utils.user_context import UserContext - from unstract.workflow_execution.enums import LogStage from workflow_manager.endpoint_v2.source import SourceConnector from workflow_manager.utils.workflow_log import WorkflowLog @@ -2434,6 +2435,7 @@ def post(self, request): # Create file history entry using the FileHistoryHelper from unstract.core.data_models import FileHashData + from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py index cbd351c91a..1f7d698468 100644 --- a/backend/workflow_manager/utils/workflow_log.py +++ b/backend/workflow_manager/utils/workflow_log.py @@ -1,10 +1,9 @@ import logging from account_v2.constants import Common -from utils.local_context import StateStore - from unstract.core.pubsub_helper import LogPublisher from unstract.workflow_execution.enums import LogComponent, LogLevel, LogStage, LogState +from utils.local_context import StateStore class WorkflowLog: diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py index d06cb06738..100968cd7f 100644 --- a/backend/workflow_manager/workflow_v2/execution.py +++ b/backend/workflow_manager/workflow_v2/execution.py @@ -8,10 +8,6 @@ from tags.models import Tag from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_processor import ToolProcessor -from usage_v2.helper import UsageHelper -from utils.local_context import StateStore -from utils.user_context import UserContext - from unstract.tool_registry.dto import Tool from unstract.workflow_execution import WorkflowExecutionService from unstract.workflow_execution.dto import ToolInstance as ToolInstanceDataClass @@ -23,6 +19,10 @@ LogState, ) from unstract.workflow_execution.exceptions import StopExecution +from usage_v2.helper import UsageHelper +from utils.local_context import StateStore +from utils.user_context import UserContext + from workflow_manager.file_execution.models import WorkflowFileExecution from workflow_manager.workflow_v2.constants import WorkflowKey from workflow_manager.workflow_v2.enums import ExecutionStatus diff --git a/backend/workflow_manager/workflow_v2/filters/execution_log.py b/backend/workflow_manager/workflow_v2/filters/execution_log.py index 2b58950207..b1a794f54d 100644 --- a/backend/workflow_manager/workflow_v2/filters/execution_log.py +++ b/backend/workflow_manager/workflow_v2/filters/execution_log.py @@ -1,8 +1,8 @@ from django.db.models.query import QuerySet from django_filters import CharFilter, FilterSet, ModelChoiceFilter from rest_framework.request import Request - from unstract.sdk1.constants import LogLevel + from workflow_manager.file_execution.models import WorkflowFileExecution from workflow_manager.workflow_v2.models.execution_log import ExecutionLog diff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.py index ed34592958..3049108bfc 100644 --- a/backend/workflow_manager/workflow_v2/serializers.py +++ b/backend/workflow_manager/workflow_v2/serializers.py @@ -1,6 +1,8 @@ import logging from typing import Any +from backend.constants import RequestKey +from backend.serializers import AuditSerializer from django.conf import settings from rest_framework.serializers import ( CharField, @@ -17,8 +19,6 @@ from utils.input_sanitizer import validate_name_field, validate_no_html_tags from utils.serializer.integrity_error_mixin import IntegrityErrorMixin -from backend.constants import RequestKey -from backend.serializers import AuditSerializer from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey from workflow_manager.workflow_v2.models.execution import WorkflowExecution from workflow_manager.workflow_v2.models.execution_log import ExecutionLog diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py index ec6fb46b77..a7374588bc 100644 --- a/backend/workflow_manager/workflow_v2/views.py +++ b/backend/workflow_manager/workflow_v2/views.py @@ -2,6 +2,7 @@ import uuid from typing import Any +from backend.constants import RequestKey from django.conf import settings from django.db import transaction from django.db.models.query import QuerySet @@ -18,11 +19,10 @@ from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning from rest_framework.views import APIView +from unstract.core.data_models import FileHistoryCreateRequest from utils.filtering import FilterHelper from utils.organization_utils import filter_queryset_by_organization, resolve_organization -from backend.constants import RequestKey -from unstract.core.data_models import FileHistoryCreateRequest from workflow_manager.endpoint_v2.destination import DestinationConnector from workflow_manager.endpoint_v2.dto import FileHash from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index 4dce7b7938..86daee381c 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -7,6 +7,7 @@ from account_v2.constants import Common from api_v2.models import APIDeployment +from backend.celery_service import app as celery_app from celery import chord, current_task from celery import exceptions as celery_exceptions from celery.result import AsyncResult @@ -20,13 +21,12 @@ from tool_instance_v2.constants import ToolInstanceKey from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_instance_helper import ToolInstanceHelper +from unstract.workflow_execution.enums import LogStage from utils.cache_service import CacheService from utils.constants import Account, CeleryQueue, FileProcessingQueue from utils.local_context import StateStore from utils.user_context import UserContext -from backend.celery_service import app as celery_app -from unstract.workflow_execution.enums import LogStage from workflow_manager.endpoint_v2.destination import DestinationConnector from workflow_manager.endpoint_v2.dto import FileHash from workflow_manager.endpoint_v2.models import WorkflowEndpoint diff --git a/docker/scripts/release-notes/print_release_notes.py b/docker/scripts/release-notes/print_release_notes.py index 6c278d88ab..b4c076e2bb 100644 --- a/docker/scripts/release-notes/print_release_notes.py +++ b/docker/scripts/release-notes/print_release_notes.py @@ -62,8 +62,7 @@ def print_release_notes(current_version, target_version, release_notes): f"{colour('WARNING', Colour.YELLOW)}: " if release_note["warning"] else "" ) message += ( - f"[from {colour(note_version, Colour.BLUE)}]\n" - f"{release_note['message']}" + f"[from {colour(note_version, Colour.BLUE)}]\n{release_note['message']}" ) messages_to_print.append(message) diff --git a/platform-service/src/unstract/platform_service/config.py b/platform-service/src/unstract/platform_service/config.py index 08fff41307..043d317c83 100644 --- a/platform-service/src/unstract/platform_service/config.py +++ b/platform-service/src/unstract/platform_service/config.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from flask import Flask - from unstract.core.flask import ( PluginManager, register_error_handlers, diff --git a/platform-service/src/unstract/platform_service/controller/__init__.py b/platform-service/src/unstract/platform_service/controller/__init__.py index 88d501cf2b..7c287fa3aa 100644 --- a/platform-service/src/unstract/platform_service/controller/__init__.py +++ b/platform-service/src/unstract/platform_service/controller/__init__.py @@ -2,11 +2,10 @@ import traceback from flask import Blueprint, Response, jsonify, request -from werkzeug.exceptions import HTTPException - from unstract.core.flask.exceptions import APIError, ErrorResponse from unstract.platform_service.controller.health import health_bp from unstract.platform_service.controller.platform import platform_bp +from werkzeug.exceptions import HTTPException api = Blueprint("api", __name__) api.register_blueprint(platform_bp) diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py index cb8dd29e06..5efbd29107 100644 --- a/platform-service/src/unstract/platform_service/controller/platform.py +++ b/platform-service/src/unstract/platform_service/controller/platform.py @@ -6,7 +6,6 @@ from cryptography.fernet import Fernet, InvalidToken from flask import Blueprint, Request, jsonify, make_response, request from flask import current_app as app - from unstract.core.flask import PluginManager from unstract.core.flask.exceptions import APIError from unstract.platform_service.constants import DBTable diff --git a/platform-service/src/unstract/platform_service/extensions.py b/platform-service/src/unstract/platform_service/extensions.py index 75a073a5fe..8613d5c211 100644 --- a/platform-service/src/unstract/platform_service/extensions.py +++ b/platform-service/src/unstract/platform_service/extensions.py @@ -4,7 +4,6 @@ import redis from peewee import PostgresqlDatabase - from unstract.core.cache.redis_client import create_redis_client db = PostgresqlDatabase(None) diff --git a/platform-service/src/unstract/platform_service/helper/cost_calculation.py b/platform-service/src/unstract/platform_service/helper/cost_calculation.py index ba2e572330..c6ff827e41 100644 --- a/platform-service/src/unstract/platform_service/helper/cost_calculation.py +++ b/platform-service/src/unstract/platform_service/helper/cost_calculation.py @@ -4,7 +4,6 @@ import requests from flask import current_app as app - from unstract.platform_service.env import Env from unstract.platform_service.utils import format_float_positional from unstract.sdk1.exceptions import FileStorageError diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py index 5edfb829ed..e167ac565b 100644 --- a/prompt-service/src/unstract/prompt_service/config.py +++ b/prompt-service/src/unstract/prompt_service/config.py @@ -5,7 +5,6 @@ from dotenv import load_dotenv from flask import Flask - from unstract.core.flask import ( PluginManager, register_error_handlers, diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py index fdf0deaae1..cc98ef1648 100644 --- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py +++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py @@ -4,7 +4,6 @@ from flask import Blueprint, request from flask import current_app as app - from unstract.core.flask import PluginManager from unstract.core.flask.exceptions import APIError from unstract.prompt_service.constants import PromptServiceConstants as PSKeys diff --git a/prompt-service/src/unstract/prompt_service/controllers/extraction.py b/prompt-service/src/unstract/prompt_service/controllers/extraction.py index 516894f429..3de32c1363 100644 --- a/prompt-service/src/unstract/prompt_service/controllers/extraction.py +++ b/prompt-service/src/unstract/prompt_service/controllers/extraction.py @@ -1,7 +1,6 @@ from typing import Any from flask import Blueprint, request - from unstract.prompt_service.constants import IndexingConstants as IKeys from unstract.prompt_service.constants import PromptServiceConstants as PSKeys from unstract.prompt_service.helpers.auth import AuthHelper diff --git a/prompt-service/src/unstract/prompt_service/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.py index f9416a9c92..5de3b0d155 100644 --- a/prompt-service/src/unstract/prompt_service/controllers/indexing.py +++ b/prompt-service/src/unstract/prompt_service/controllers/indexing.py @@ -2,7 +2,6 @@ from typing import Any from flask import Blueprint, request - from unstract.prompt_service.constants import IndexingConstants as IKeys from unstract.prompt_service.constants import PromptServiceConstants as PSKeys from unstract.prompt_service.dto import ( diff --git a/prompt-service/src/unstract/prompt_service/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.py index 5f1703b320..167b7c6d72 100644 --- a/prompt-service/src/unstract/prompt_service/core/index_v2.py +++ b/prompt-service/src/unstract/prompt_service/core/index_v2.py @@ -11,7 +11,6 @@ VectorStoreQuery, VectorStoreQueryResult, ) - from unstract.prompt_service.dto import ( ChunkingConfig, FileInfo, diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py b/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py index 3df2a3776e..55a807901d 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py @@ -3,7 +3,6 @@ from llama_index.core import VectorStoreIndex from llama_index.core.retrievers import AutoMergingRetriever as LlamaAutoMergingRetriever from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py index 5fd7870f01..244c187849 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py @@ -3,7 +3,6 @@ from llama_index.core import VectorStoreIndex from llama_index.core.retrievers import QueryFusionRetriever from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py b/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py index d17e507b50..b80149151f 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py @@ -3,7 +3,6 @@ from llama_index.core import VectorStoreIndex from llama_index.core.indices.keyword_table import KeywordTableIndex from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py b/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py index dcec3d6176..5069fd9b12 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py @@ -3,7 +3,6 @@ from llama_index.core import VectorStoreIndex from llama_index.core.retrievers import RecursiveRetriever from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py b/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py index c2038f9181..36285271ee 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py @@ -14,7 +14,6 @@ ) from llama_index.core.llms.llm import LLM as LlamaIndexBaseLLM # noqa: N811 from pydantic import PrivateAttr - from unstract.sdk1.llm import LLM, LLMCompat diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/router.py b/prompt-service/src/unstract/prompt_service/core/retrievers/router.py index b5bc4efe1d..2a315c5230 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/router.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/router.py @@ -5,7 +5,6 @@ from llama_index.core.selectors import LLMSingleSelector from llama_index.core.tools import QueryEngineTool, ToolMetadata from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py index 15f25db1bb..76a4984109 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py @@ -3,7 +3,6 @@ from flask import current_app as app from llama_index.core import VectorStoreIndex from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py index 6930bc131b..10703e9bc6 100644 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py +++ b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py @@ -4,7 +4,6 @@ from llama_index.core.question_gen.llm_generators import LLMQuestionGenerator from llama_index.core.schema import QueryBundle from llama_index.core.tools import QueryEngineTool, ToolMetadata - from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever from unstract.prompt_service.exceptions import RetrievalError diff --git a/prompt-service/src/unstract/prompt_service/extensions.py b/prompt-service/src/unstract/prompt_service/extensions.py index 1626591dc0..8fc934a552 100644 --- a/prompt-service/src/unstract/prompt_service/extensions.py +++ b/prompt-service/src/unstract/prompt_service/extensions.py @@ -4,7 +4,6 @@ from typing import Any from playhouse.postgres_ext import PostgresqlExtDatabase - from unstract.prompt_service.utils.env_loader import get_env_or_die # Load required environment variables diff --git a/prompt-service/src/unstract/prompt_service/helpers/auth.py b/prompt-service/src/unstract/prompt_service/helpers/auth.py index 3c74e5abe1..87159a1898 100644 --- a/prompt-service/src/unstract/prompt_service/helpers/auth.py +++ b/prompt-service/src/unstract/prompt_service/helpers/auth.py @@ -2,7 +2,6 @@ from flask import Request, request from flask import current_app as app - from unstract.prompt_service.constants import DBTableV2 from unstract.prompt_service.extensions import db, db_context from unstract.prompt_service.utils.db_utils import DBUtils diff --git a/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py index e2a30a0e2e..6d6ac21856 100644 --- a/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py +++ b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py @@ -2,7 +2,6 @@ import warnings from flask import current_app - from unstract.prompt_service.constants import PromptServiceConstants from unstract.sdk1.constants import LogLevel from unstract.sdk1.tool.stream import StreamMixin diff --git a/prompt-service/src/unstract/prompt_service/helpers/usage.py b/prompt-service/src/unstract/prompt_service/helpers/usage.py index 63a14c4336..0b0a27ffae 100644 --- a/prompt-service/src/unstract/prompt_service/helpers/usage.py +++ b/prompt-service/src/unstract/prompt_service/helpers/usage.py @@ -4,7 +4,6 @@ from typing import Any from flask import current_app as app - from unstract.prompt_service.constants import DBTableV2 from unstract.prompt_service.extensions import db, db_context from unstract.prompt_service.utils.db_utils import DBUtils diff --git a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py index cd794ce590..4f162f42a2 100644 --- a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py +++ b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py @@ -4,7 +4,6 @@ from typing import Any from flask import current_app as app - from unstract.prompt_service.constants import VariableConstants, VariableType from unstract.prompt_service.exceptions import CustomDataError from unstract.prompt_service.utils.request import HTTPMethod, make_http_request diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py index 9f8cbf9c28..bc72ab0370 100644 --- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py +++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py @@ -5,7 +5,6 @@ from urllib.parse import urlparse from flask import current_app as app - from unstract.core.flask import PluginManager from unstract.core.flask.exceptions import APIError from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys, RunLevel diff --git a/prompt-service/src/unstract/prompt_service/services/retrieval.py b/prompt-service/src/unstract/prompt_service/services/retrieval.py index 92ba46c930..6ef38dc308 100644 --- a/prompt-service/src/unstract/prompt_service/services/retrieval.py +++ b/prompt-service/src/unstract/prompt_service/services/retrieval.py @@ -2,7 +2,6 @@ from typing import Any from flask import current_app as app - from unstract.prompt_service.constants import PromptServiceConstants as PSKeys from unstract.prompt_service.constants import RetrievalStrategy from unstract.prompt_service.core.retrievers.automerging import AutomergingRetriever diff --git a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py index 41ba342ac2..5d76645f46 100644 --- a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py +++ b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py @@ -1,7 +1,6 @@ from typing import Any from flask import current_app as app - from unstract.prompt_service.constants import PromptServiceConstants as PSKeys from unstract.prompt_service.constants import RunLevel, VariableType from unstract.prompt_service.helpers.variable_replacement import ( diff --git a/prompt-service/src/unstract/prompt_service/utils/request.py b/prompt-service/src/unstract/prompt_service/utils/request.py index 8a5a82d462..62cba05d8c 100644 --- a/prompt-service/src/unstract/prompt_service/utils/request.py +++ b/prompt-service/src/unstract/prompt_service/utils/request.py @@ -4,7 +4,6 @@ import requests as pyrequests from flask import current_app as app from requests.exceptions import RequestException - from unstract.prompt_service.exceptions import APIError, BadRequest, MissingFieldError diff --git a/runner/src/unstract/runner/__init__.py b/runner/src/unstract/runner/__init__.py index 54041b09fc..b54d23871e 100644 --- a/runner/src/unstract/runner/__init__.py +++ b/runner/src/unstract/runner/__init__.py @@ -2,7 +2,6 @@ import os from flask import Flask - from unstract.core.flask import register_error_handlers, register_request_id_middleware from unstract.core.flask.logging import setup_logging from unstract.runner.constants import LogLevel diff --git a/runner/src/unstract/runner/clients/docker_client.py b/runner/src/unstract/runner/clients/docker_client.py index 735f0f3415..69c635eada 100644 --- a/runner/src/unstract/runner/clients/docker_client.py +++ b/runner/src/unstract/runner/clients/docker_client.py @@ -4,7 +4,6 @@ from collections.abc import Iterator from typing import Any -from docker import DockerClient from docker.errors import APIError, ImageNotFound, NotFound from docker.models.containers import Container from unstract.core.runner.enum import ContainerStatus @@ -17,6 +16,8 @@ from unstract.runner.exception import ToolImageNotFoundError from unstract.runner.utils import Utils +from docker import DockerClient + class DockerContainer(ContainerInterface): def __init__(self, container: Container, logger: logging.Logger) -> None: diff --git a/runner/src/unstract/runner/clients/test_docker.py b/runner/src/unstract/runner/clients/test_docker.py index 1c1418b000..093f086a0f 100644 --- a/runner/src/unstract/runner/clients/test_docker.py +++ b/runner/src/unstract/runner/clients/test_docker.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock import pytest - from docker.errors import ImageNotFound from unstract.runner.constants import Env diff --git a/runner/src/unstract/runner/controller/run.py b/runner/src/unstract/runner/controller/run.py index dc681630c1..692e26aba8 100644 --- a/runner/src/unstract/runner/controller/run.py +++ b/runner/src/unstract/runner/controller/run.py @@ -2,7 +2,6 @@ from flask import Blueprint, abort, request from flask import current_app as app - from unstract.runner.runner import UnstractRunner # Define a Blueprint with a root URL path diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py index 1125bf69e8..e5d361e029 100644 --- a/runner/src/unstract/runner/runner.py +++ b/runner/src/unstract/runner/runner.py @@ -7,7 +7,6 @@ from dotenv import load_dotenv from flask import Flask - from unstract.core.constants import LogFieldName from unstract.core.file_execution_tracker import ( FileExecutionStage, diff --git a/runner/src/unstract/runner/utils.py b/runner/src/unstract/runner/utils.py index 4bec83a2cb..d021feebd8 100644 --- a/runner/src/unstract/runner/utils.py +++ b/runner/src/unstract/runner/utils.py @@ -2,7 +2,6 @@ import os from dotenv import load_dotenv - from unstract.runner.constants import Env from unstract.runner.enum import LogLevel diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.py index 4903d727b2..77ff9356a1 100644 --- a/tools/classifier/src/main.py +++ b/tools/classifier/src/main.py @@ -5,7 +5,6 @@ ClassifierHelper, # type: ignore ReservedBins, ) - from unstract.sdk1.constants import ( LogLevel, LogState, diff --git a/tools/structure/src/helpers.py b/tools/structure/src/helpers.py index 2cb0d9bf72..a08afa2f97 100644 --- a/tools/structure/src/helpers.py +++ b/tools/structure/src/helpers.py @@ -5,7 +5,6 @@ from constants import IndexingConstants as IKeys from constants import SettingsKeys # type: ignore [attr-defined] - from unstract.sdk1.prompt import PromptTool from unstract.sdk1.tool.base import BaseTool diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py index f68143a6c8..b16148a064 100644 --- a/tools/structure/src/main.py +++ b/tools/structure/src/main.py @@ -8,13 +8,12 @@ from constants import SettingsKeys # type: ignore [attr-defined] from helpers import StructureToolHelper as STHelper -from utils import json_to_markdown, repair_json_with_best_structure - from unstract.sdk1.constants import LogState, MetadataKey, ToolEnv, UsageKwargs from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.prompt import PromptTool from unstract.sdk1.tool.base import BaseTool from unstract.sdk1.tool.entrypoint import ToolEntrypoint +from utils import json_to_markdown, repair_json_with_best_structure logger = logging.getLogger(__name__) diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py index f73e3b731c..e99937de39 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py @@ -272,8 +272,7 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to Google Drive, " - "please check the connection settings." + "Unable to connect to Google Drive, please check the connection settings." ) return True diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py index 87e786a7a3..a3350b9c94 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py @@ -119,7 +119,6 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to HTTP server, " - "please check the connection settings." + "Unable to connect to HTTP server, please check the connection settings." ) return True diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py index 34cfa79659..4fb1943746 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py @@ -4,7 +4,6 @@ from typing import Any from fsspec.implementations.local import LocalFileSystem - from unstract.connectors.exceptions import ConnectorError from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py index 3e3915debc..e0f3989027 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py @@ -140,8 +140,7 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to SFTP server, " - "please check the connection settings." + "Unable to connect to SFTP server, please check the connection settings." ) return True diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py index 870ed30a02..303138b206 100644 --- a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py +++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py @@ -18,8 +18,7 @@ def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider: except KeyError: allowed_providers = ", ".join([provider.name for provider in FileStorageProvider]) logger.error( - f"Invalid provider '{provider_name}'. Allowed providers: " - f"{allowed_providers}" + f"Invalid provider '{provider_name}'. Allowed providers: {allowed_providers}" ) raise ProviderNotFound(f"Provider '{provider_name}' not found") diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py b/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py index 55227e662d..f4cae34cbc 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py @@ -5,6 +5,7 @@ from typing import Any from singleton_decorator import singleton + from unstract.sdk1.adapters import AdapterDict from unstract.sdk1.adapters.base import Adapter from unstract.sdk1.adapters.constants import Common diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py index 8ad721c3d4..8b41ff45d8 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py @@ -10,6 +10,7 @@ from typing import Any from pydantic import BaseModel, Field, model_validator + from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.enums import AdapterTypes diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py b/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py index a36d3ae357..327db3b89b 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py @@ -8,6 +8,7 @@ from filetype import filetype from google.auth.transport import requests as google_requests from google.oauth2.service_account import Credentials + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.ocr.constants import FileType from unstract.sdk1.adapters.ocr.ocr_adapter import OCRAdapter diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py index edef6bd043..a1c2b76a4b 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py @@ -1,4 +1,5 @@ from qdrant_client.http.exceptions import ApiException as QdrantAPIException + from unstract.sdk1.adapters.vectordb.qdrant.src import Qdrant from unstract.sdk1.adapters.vectordb.vectordb_adapter import VectorDBAdapter from unstract.sdk1.exceptions import VectorDBError diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py index ed2b4921e9..5e77d5da9d 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py @@ -9,6 +9,7 @@ ) from llama_index.core.llms import MockLLM from llama_index.core.vector_stores.types import BasePydanticVectorStore + from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.exceptions import VectorDBError diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py index 35a7191b78..ddca558e45 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from llama_index.vector_stores.milvus import MilvusVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py index 0f396a79e5..edbbc76716 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py @@ -3,6 +3,7 @@ from llama_index.core.schema import BaseNode from llama_index.core.vector_stores.types import VectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py index 17fe6dcbcc..2f2cbbdad0 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py @@ -6,6 +6,7 @@ from llama_index.vector_stores.pinecone import PineconeVectorStore from pinecone import NotFoundException, PodSpec, ServerlessSpec from pinecone import Pinecone as LLamaIndexPinecone + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py index bfe28823cc..1e4527dc68 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py @@ -6,6 +6,7 @@ import psycopg2 from llama_index.vector_stores.postgres import PGVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py index afd9be0c3a..e5dac11d58 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py @@ -5,6 +5,7 @@ from llama_index.vector_stores.qdrant import QdrantVectorStore from qdrant_client import QdrantClient from qdrant_client.http.exceptions import UnexpectedResponse + from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper from unstract.sdk1.adapters.vectordb.vectordb_adapter import VectorDBAdapter diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py index 6508eaad93..347fd6c729 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py @@ -5,6 +5,7 @@ from urllib.parse import quote_plus from llama_index.vector_stores.supabase import SupabaseVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py index 786ab11457..9ccee85ba3 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py @@ -4,6 +4,7 @@ from llama_index.core.schema import BaseNode from llama_index.core.vector_stores import SimpleVectorStore from llama_index.core.vector_stores.types import BasePydanticVectorStore, VectorStore + from unstract.sdk1.adapters.base import Adapter from unstract.sdk1.adapters.enums import AdapterTypes from unstract.sdk1.exceptions import VectorDBError diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py index 457f80a073..f9d851438a 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py @@ -4,12 +4,13 @@ import weaviate from llama_index.core.vector_stores.types import BasePydanticVectorStore from llama_index.vector_stores.weaviate import WeaviateVectorStore +from weaviate.classes.init import Auth +from weaviate.exceptions import UnexpectedStatusCodeException + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper from unstract.sdk1.adapters.vectordb.vectordb_adapter import VectorDBAdapter -from weaviate.classes.init import Auth -from weaviate.exceptions import UnexpectedStatusCodeException logger = logging.getLogger(__name__) diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py index 0f4f2f196b..6792fe739a 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py @@ -4,6 +4,7 @@ import requests from requests import Response from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py index 19a49c31d5..4d969f9db9 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py @@ -5,6 +5,7 @@ from httpx import ConnectError from llama_parse import LlamaParse + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.x2text.dto import TextExtractionResult from unstract.sdk1.adapters.x2text.llama_parse.src.constants import LlamaParseConfig diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py index 1d8f1ee388..abecb4ce64 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py @@ -8,6 +8,7 @@ import requests from requests import Response from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.adapters.exceptions import ExtractorError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py index 7d0983673d..443acc423b 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py @@ -11,6 +11,7 @@ LLMWhispererClientException, LLMWhispererClientV2, ) + from unstract.sdk1.adapters.exceptions import ExtractorError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/audit.py b/unstract/sdk1/src/unstract/sdk1/audit.py index 0396ab33a8..e1d960f43f 100644 --- a/unstract/sdk1/src/unstract/sdk1/audit.py +++ b/unstract/sdk1/src/unstract/sdk1/audit.py @@ -2,6 +2,7 @@ import requests from llama_index.core.callbacks import CBEventType, TokenCountingHandler + from unstract.sdk1.constants import LogLevel, ToolEnv from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.stream import StreamMixin diff --git a/unstract/sdk1/src/unstract/sdk1/cache.py b/unstract/sdk1/src/unstract/sdk1/cache.py index 9a85659209..ab22a9e54d 100644 --- a/unstract/sdk1/src/unstract/sdk1/cache.py +++ b/unstract/sdk1/src/unstract/sdk1/cache.py @@ -1,4 +1,5 @@ import requests + from unstract.sdk1.constants import LogLevel from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.base import BaseTool diff --git a/unstract/sdk1/src/unstract/sdk1/embedding.py b/unstract/sdk1/src/unstract/sdk1/embedding.py index e54a093393..b32ea8cf98 100644 --- a/unstract/sdk1/src/unstract/sdk1/embedding.py +++ b/unstract/sdk1/src/unstract/sdk1/embedding.py @@ -4,9 +4,10 @@ from typing import TYPE_CHECKING import litellm -import unstract.sdk1.patches.litellm_cohere_timeout # noqa: F401 from llama_index.core.embeddings import BaseEmbedding from pydantic import ValidationError + +import unstract.sdk1.patches.litellm_cohere_timeout # noqa: F401 from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.embedding1 import adapters from unstract.sdk1.constants import Common as SdkCommon diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py index 8fdf7e1069..4a343da7d9 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py @@ -2,6 +2,7 @@ import fsspec from fsspec import AbstractFileSystem + from unstract.sdk1.exceptions import FileOperationError, FileStorageError from unstract.sdk1.file_storage.provider import FileStorageProvider diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py b/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py index 9666c86524..7df347c931 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py @@ -7,6 +7,7 @@ import fsspec import magic import yaml + from unstract.sdk1.exceptions import FileOperationError from unstract.sdk1.file_storage.constants import FileOperationParams, FileSeekPosition from unstract.sdk1.file_storage.helper import FileStorageHelper, skip_local_cache diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py b/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py index d053f149f5..c3a171cf7f 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py @@ -3,6 +3,7 @@ from typing import Any from fsspec import AbstractFileSystem + from unstract.sdk1.file_storage.constants import FileOperationParams, FileSeekPosition diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py b/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py index a21fe54315..0d5ce03eca 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py @@ -3,6 +3,7 @@ import filetype import magic + from unstract.sdk1.exceptions import FileOperationError, FileStorageError from unstract.sdk1.file_storage.constants import FileOperationParams from unstract.sdk1.file_storage.impl import FileStorage diff --git a/unstract/sdk1/src/unstract/sdk1/index.py b/unstract/sdk1/src/unstract/sdk1/index.py index 92220f0a87..01e27c5f3f 100644 --- a/unstract/sdk1/src/unstract/sdk1/index.py +++ b/unstract/sdk1/src/unstract/sdk1/index.py @@ -13,6 +13,7 @@ VectorStoreQuery, VectorStoreQueryResult, ) + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.no_op.src.no_op_custom_vectordb import ( NoOpCustomVectorDB, diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py index 8ff29a89d5..1028f6c379 100644 --- a/unstract/sdk1/src/unstract/sdk1/llm.py +++ b/unstract/sdk1/src/unstract/sdk1/llm.py @@ -11,6 +11,7 @@ # from litellm import get_supported_openai_params from litellm import get_max_tokens, token_counter from pydantic import ValidationError + from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.llm1 import adapters from unstract.sdk1.audit import Audit diff --git a/unstract/sdk1/src/unstract/sdk1/platform.py b/unstract/sdk1/src/unstract/sdk1/platform.py index 2a3b75a97b..8498ea6d32 100644 --- a/unstract/sdk1/src/unstract/sdk1/platform.py +++ b/unstract/sdk1/src/unstract/sdk1/platform.py @@ -5,6 +5,7 @@ import requests from requests import RequestException, Response from requests.exceptions import ConnectionError, HTTPError + from unstract.sdk1.constants import ( AdapterKeys, Common, diff --git a/unstract/sdk1/src/unstract/sdk1/prompt.py b/unstract/sdk1/src/unstract/sdk1/prompt.py index c648506209..d39ee68df7 100644 --- a/unstract/sdk1/src/unstract/sdk1/prompt.py +++ b/unstract/sdk1/src/unstract/sdk1/prompt.py @@ -5,6 +5,7 @@ import requests from requests import ConnectionError, RequestException, Response + from unstract.sdk1.constants import MimeType, RequestHeader, ToolEnv from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.base import BaseTool diff --git a/unstract/sdk1/src/unstract/sdk1/tool/cache.py b/unstract/sdk1/src/unstract/sdk1/tool/cache.py index 159f905c50..737787960e 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/cache.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/cache.py @@ -1,4 +1,5 @@ import requests + from unstract.sdk1.constants import LogLevel from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.base import BaseTool diff --git a/unstract/sdk1/src/unstract/sdk1/tool/parser.py b/unstract/sdk1/src/unstract/sdk1/tool/parser.py index ed439c2615..787c49ebf0 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/parser.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/parser.py @@ -1,6 +1,7 @@ import argparse from dotenv import find_dotenv, load_dotenv + from unstract.sdk1.constants import LogLevel diff --git a/unstract/sdk1/src/unstract/sdk1/tool/stream.py b/unstract/sdk1/src/unstract/sdk1/tool/stream.py index 92e3b67356..6cc387d331 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/stream.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/stream.py @@ -5,6 +5,7 @@ from typing import Any from deprecated import deprecated + from unstract.sdk1.constants import Command, LogLevel, LogStage, ToolEnv from unstract.sdk1.exceptions import SdkError from unstract.sdk1.utils.common import Utils diff --git a/unstract/sdk1/src/unstract/sdk1/tool/validator.py b/unstract/sdk1/src/unstract/sdk1/tool/validator.py index 30d6ae475e..b55bf9a555 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/validator.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/validator.py @@ -3,6 +3,7 @@ from pathlib import Path from jsonschema import Draft202012Validator, ValidationError, validators + from unstract.sdk1.constants import MetadataKey, PropKey from unstract.sdk1.tool.base import BaseTool from unstract.sdk1.tool.mime_types import EXT_MIME_MAP diff --git a/unstract/sdk1/src/unstract/sdk1/usage_handler.py b/unstract/sdk1/src/unstract/sdk1/usage_handler.py index 0ffa7c43e2..874f26cf6a 100644 --- a/unstract/sdk1/src/unstract/sdk1/usage_handler.py +++ b/unstract/sdk1/src/unstract/sdk1/usage_handler.py @@ -3,6 +3,7 @@ from llama_index.core.callbacks import CBEventType, TokenCountingHandler from llama_index.core.callbacks.base_handler import BaseCallbackHandler from llama_index.core.embeddings import BaseEmbedding + from unstract.sdk1.audit import Audit from unstract.sdk1.constants import LogLevel from unstract.sdk1.tool.stream import StreamMixin diff --git a/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py b/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py index b5a1c5578f..2bcdad78cb 100644 --- a/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py +++ b/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py @@ -7,6 +7,7 @@ from llama_index.core.callbacks import TokenCountingHandler from llama_index.core.embeddings import BaseEmbedding from llama_index.core.llms import LLM + from unstract.sdk1.usage_handler import UsageHandler logger = logging.getLogger(__name__) diff --git a/unstract/sdk1/src/unstract/sdk1/vector_db.py b/unstract/sdk1/src/unstract/sdk1/vector_db.py index 9225fb8002..8ffd8513d1 100644 --- a/unstract/sdk1/src/unstract/sdk1/vector_db.py +++ b/unstract/sdk1/src/unstract/sdk1/vector_db.py @@ -11,6 +11,7 @@ VectorStore, VectorStoreQueryResult, ) + from unstract.sdk1.adapters.vectordb import adapters from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.exceptions import parse_vector_db_err diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.py index 2024f8cdbc..8d4127344e 100644 --- a/unstract/sdk1/src/unstract/sdk1/x2txt.py +++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py @@ -2,6 +2,7 @@ from typing import Any import pdfplumber + from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.x2text import adapters from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py b/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py index dc9f7c7517..70b84b4349 100644 --- a/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py +++ b/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py @@ -4,6 +4,7 @@ import httpx import pytest + from unstract.sdk1.patches.litellm_cohere_timeout import ( _patched_async_embedding, _patched_embedding, @@ -105,12 +106,12 @@ def test_none_timeout_passed_to_client_post( ) call_kwargs = mock_http_handler.post.call_args - assert ( - "timeout" in call_kwargs.kwargs - ), "timeout kwarg must always be passed to client.post()" - assert ( - call_kwargs.kwargs["timeout"] is None - ), f"Expected timeout=None, got timeout={call_kwargs.kwargs['timeout']}" + assert "timeout" in call_kwargs.kwargs, ( + "timeout kwarg must always be passed to client.post()" + ) + assert call_kwargs.kwargs["timeout"] is None, ( + f"Expected timeout=None, got timeout={call_kwargs.kwargs['timeout']}" + ) def test_httpx_timeout_object_forwarded( self, diff --git a/unstract/sdk1/tests/test_llm_compat.py b/unstract/sdk1/tests/test_llm_compat.py index c9eac4095d..d7d3a6b17e 100644 --- a/unstract/sdk1/tests/test_llm_compat.py +++ b/unstract/sdk1/tests/test_llm_compat.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest + from unstract.sdk1.llm import ( LLM, ChatMessage, @@ -358,7 +359,7 @@ def test_multi_turn_conversation(self: Self) -> None: ] result = LLMCompat._messages_to_prompt(messages) assert result == ( - "system: Be concise\n" "user: First\n" "assistant: Answer\n" "user: Follow-up" + "system: Be concise\nuser: First\nassistant: Answer\nuser: Follow-up" ) def test_empty_messages_returns_empty_string(self: Self) -> None: diff --git a/unstract/sdk1/tests/test_platform.py b/unstract/sdk1/tests/test_platform.py index f318175063..b390f94562 100644 --- a/unstract/sdk1/tests/test_platform.py +++ b/unstract/sdk1/tests/test_platform.py @@ -5,6 +5,7 @@ import pytest from _pytest.monkeypatch import MonkeyPatch from requests.exceptions import ConnectionError, HTTPError + from unstract.sdk1.exceptions import SdkError from unstract.sdk1.platform import PlatformHelper diff --git a/unstract/sdk1/tests/test_prompt.py b/unstract/sdk1/tests/test_prompt.py index 798db9106c..b4350fd3f2 100644 --- a/unstract/sdk1/tests/test_prompt.py +++ b/unstract/sdk1/tests/test_prompt.py @@ -6,6 +6,7 @@ import pytest from pytest import MonkeyPatch from requests.exceptions import ConnectionError, Timeout + from unstract.sdk1.prompt import PromptTool diff --git a/unstract/sdk1/tests/utils/test_retry_utils.py b/unstract/sdk1/tests/utils/test_retry_utils.py index cbf73c731a..785093f2f2 100644 --- a/unstract/sdk1/tests/utils/test_retry_utils.py +++ b/unstract/sdk1/tests/utils/test_retry_utils.py @@ -7,6 +7,7 @@ import pytest from _pytest.monkeypatch import MonkeyPatch from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.utils.retry_utils import ( calculate_delay, create_retry_decorator, diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 6406a864e6..1fcb7619b5 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -22,10 +22,9 @@ from shared.processing.files import FileProcessingUtils from shared.workflow.execution import WorkerExecutionContext, WorkflowOrchestrationUtils from shared.workflow.execution.tool_validation import validate_workflow_tool_instances -from worker import app - from unstract.core.data_models import ExecutionStatus, FileHashData, WorkerFileData from unstract.core.worker_models import ApiDeploymentResultStatus +from worker import app logger = WorkerLogger.get_logger(__name__) diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py index d7252b9d29..cd3af597f0 100644 --- a/workers/callback/tasks.py +++ b/workers/callback/tasks.py @@ -1025,9 +1025,10 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any, Raises: ValueError: If execution type cannot be determined """ - from unstract.filesystem import FileStorageType from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler + from unstract.filesystem import FileStorageType + # Determine if this is an API or workflow execution is_api_execution = context.pipeline_data and context.pipeline_data.get( "is_api", False diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py index 78dbcc3ede..a004072a73 100644 --- a/workers/file_processing/tasks.py +++ b/workers/file_processing/tasks.py @@ -39,10 +39,6 @@ create_organization_context, ) from shared.processing.files.processor import FileProcessor - -# Import manual review service with WorkflowUtil access -from worker import app - from unstract.core.data_models import ( ExecutionStatus, FileBatchData, @@ -53,6 +49,9 @@ ) from unstract.core.worker_models import FileProcessingResult +# Import manual review service with WorkflowUtil access +from worker import app + logger = WorkerLogger.get_logger(__name__) # Constants diff --git a/workers/general/tasks.py b/workers/general/tasks.py index addf18b9a6..98707dfc7d 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -44,10 +44,6 @@ ) from shared.workflow.execution.tool_validation import validate_workflow_tool_instances -# File management handled by StreamingFileDiscovery -# Import from local worker module (avoid circular import) -from worker import app, config - # Import shared data models for type safety from unstract.core.data_models import ( ExecutionStatus, @@ -59,6 +55,10 @@ # Import common workflow utilities from unstract.core.workflow_utils import WorkflowTypeDetector +# File management handled by StreamingFileDiscovery +# Import from local worker module (avoid circular import) +from worker import app, config + logger = WorkerLogger.get_logger(__name__) diff --git a/workers/log_consumer/process_log_history.py b/workers/log_consumer/process_log_history.py index a941bb0609..8a534633c5 100755 --- a/workers/log_consumer/process_log_history.py +++ b/workers/log_consumer/process_log_history.py @@ -17,7 +17,6 @@ import sys import httpx - from unstract.core.cache.redis_queue_client import RedisQueueClient logger = logging.getLogger(__name__) diff --git a/workers/log_consumer/tasks.py b/workers/log_consumer/tasks.py index 3eaa280858..ad6a72039d 100644 --- a/workers/log_consumer/tasks.py +++ b/workers/log_consumer/tasks.py @@ -12,7 +12,6 @@ from shared.infrastructure.config import WorkerConfig from shared.infrastructure.logging import WorkerLogger from shared.utils.api_client_singleton import get_singleton_api_client - from unstract.core.cache.redis_queue_client import RedisQueueClient from unstract.core.constants import LogEventArgument, LogProcessingTask from unstract.core.log_utils import store_execution_log diff --git a/workers/notification/providers/registry.py b/workers/notification/providers/registry.py index b6f4c58ac9..43b9a82da3 100644 --- a/workers/notification/providers/registry.py +++ b/workers/notification/providers/registry.py @@ -9,7 +9,6 @@ from notification.providers.base_provider import BaseNotificationProvider from notification.providers.slack_webhook import SlackWebhook from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_enums import NotificationType logger = WorkerLogger.get_logger(__name__) diff --git a/workers/notification/providers/webhook_provider.py b/workers/notification/providers/webhook_provider.py index 57a6382d55..8675151768 100644 --- a/workers/notification/providers/webhook_provider.py +++ b/workers/notification/providers/webhook_provider.py @@ -13,7 +13,6 @@ ValidationError, ) from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_utils import ( build_webhook_headers, send_webhook_request, diff --git a/workers/notification/tasks.py b/workers/notification/tasks.py index 32a684ad9c..4b60360633 100644 --- a/workers/notification/tasks.py +++ b/workers/notification/tasks.py @@ -24,7 +24,6 @@ ) from shared.infrastructure.config import WorkerConfig from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_enums import NotificationType logger = WorkerLogger.get_logger(__name__) diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py index 2e22946a58..47053b18fb 100644 --- a/workers/scheduler/tasks.py +++ b/workers/scheduler/tasks.py @@ -21,7 +21,6 @@ ) from shared.patterns.notification.helper import trigger_notification from shared.utils.api_client_singleton import get_singleton_api_client - from unstract.core.data_models import NotificationPayload, NotificationSource # Import the exact backend logic to ensure consistency diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py index def90bd86a..a47196fd34 100644 --- a/workers/shared/api/internal_client.py +++ b/workers/shared/api/internal_client.py @@ -58,7 +58,6 @@ WebhookNotificationRequest, ) from shared.models.scheduler_models import SchedulerExecutionResult - from unstract.core.data_models import ( ExecutionStatus, FileExecutionCreateRequest, diff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.py index a7ba8c7013..eb1ef51454 100644 --- a/workers/shared/clients/usage_client.py +++ b/workers/shared/clients/usage_client.py @@ -16,7 +16,6 @@ from uuid import UUID from shared.data.response_models import APIResponse, ResponseStatus - from unstract.core.data_models import UsageResponseData from .base_client import BaseAPIClient diff --git a/workers/shared/clients/workflow_client.py b/workers/shared/clients/workflow_client.py index 42c1ab549f..3faabac809 100644 --- a/workers/shared/clients/workflow_client.py +++ b/workers/shared/clients/workflow_client.py @@ -19,7 +19,6 @@ from uuid import UUID from shared.data.response_models import APIResponse, ResponseStatus - from unstract.core.data_models import ( WorkflowDefinitionResponseData, WorkflowEndpointConfigResponseData, diff --git a/workers/shared/infrastructure/caching/cache_utils.py b/workers/shared/infrastructure/caching/cache_utils.py index 28441a6fc9..05fdfd0175 100644 --- a/workers/shared/infrastructure/caching/cache_utils.py +++ b/workers/shared/infrastructure/caching/cache_utils.py @@ -9,7 +9,6 @@ import time from shared.infrastructure.config import WorkerConfig - from unstract.core.cache.redis_client import create_redis_client logger = logging.getLogger(__name__) diff --git a/workers/shared/infrastructure/config/worker_config.py b/workers/shared/infrastructure/config/worker_config.py index e1e11997c2..b3d29afa5f 100644 --- a/workers/shared/infrastructure/config/worker_config.py +++ b/workers/shared/infrastructure/config/worker_config.py @@ -247,14 +247,14 @@ class WorkerConfig: # by directly assigning shared_session to client.session, which breaks fork detection. # Only enable if you understand the fork-safety implications and are not using prefork pool. enable_api_client_singleton: bool = field( - default_factory=lambda: os.getenv("ENABLE_API_CLIENT_SINGLETON", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_API_CLIENT_SINGLETON", "false").lower() == "true" + ) ) enable_organization_context_cache: bool = field( - default_factory=lambda: os.getenv( - "ENABLE_ORGANIZATION_CONTEXT_CACHE", "true" - ).lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_ORGANIZATION_CONTEXT_CACHE", "true").lower() == "true" + ) ) api_client_pool_size: int = field( default_factory=lambda: int(os.getenv("API_CLIENT_POOL_SIZE", "10")) @@ -277,16 +277,19 @@ class WorkerConfig: # Debug Logging Control (Performance Optimization) enable_debug_logging: bool = field( - default_factory=lambda: os.getenv("ENABLE_DEBUG_LOGGING", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_DEBUG_LOGGING", "false").lower() == "true" + ) ) debug_api_client_init: bool = field( - default_factory=lambda: os.getenv("DEBUG_API_CLIENT_INIT", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("DEBUG_API_CLIENT_INIT", "false").lower() == "true" + ) ) debug_organization_context: bool = field( - default_factory=lambda: os.getenv("DEBUG_ORGANIZATION_CONTEXT", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("DEBUG_ORGANIZATION_CONTEXT", "false").lower() == "true" + ) ) # Task Timeout Settings (in seconds) @@ -301,8 +304,9 @@ class WorkerConfig: default_factory=lambda: os.getenv("ENABLE_METRICS", "true").lower() == "true" ) enable_health_server: bool = field( - default_factory=lambda: os.getenv("ENABLE_HEALTH_SERVER", "true").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_HEALTH_SERVER", "true").lower() == "true" + ) ) metrics_port: int = field( default_factory=lambda: int(os.getenv("METRICS_PORT", "8080")) diff --git a/workers/shared/infrastructure/logging/logger.py b/workers/shared/infrastructure/logging/logger.py index ea0d1f3d01..514275fabe 100644 --- a/workers/shared/infrastructure/logging/logger.py +++ b/workers/shared/infrastructure/logging/logger.py @@ -623,7 +623,9 @@ def ensure_dict_access(obj: Any, keys: list, default: Any = None) -> dict[str, A Dictionary with extracted values Example: - >>> ensure_dict_access(file_data, ["execution_id", "workflow_id", "organization_id"]) + >>> ensure_dict_access( + ... file_data, ["execution_id", "workflow_id", "organization_id"] + ... ) {'execution_id': '123', 'workflow_id': '456', 'organization_id': 'org_789'} """ return {key: safe_get_attr(obj, key, default) for key in keys} diff --git a/workers/shared/models/file_processing.py b/workers/shared/models/file_processing.py index af40624176..613139646c 100644 --- a/workers/shared/models/file_processing.py +++ b/workers/shared/models/file_processing.py @@ -12,7 +12,6 @@ # from unstract.api.internal_client import InternalAPIClient # from unstract.infrastructure.logging.workflow_logger import WorkerWorkflowLogger from shared.infrastructure.logging import WorkerLogger - from unstract.core.data_models import FileHashData, WorkerFileData logger = WorkerLogger.get_logger(__name__) diff --git a/workers/shared/models/result_models.py b/workers/shared/models/result_models.py index 547145b922..d82b3b997a 100644 --- a/workers/shared/models/result_models.py +++ b/workers/shared/models/result_models.py @@ -12,9 +12,9 @@ # Import shared domain models from core sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../unstract/core/src")) from shared.enums import QueueResultStatus +from unstract.core.worker_models import FileExecutionResult from unstract.core import ExecutionStatus, serialize_dataclass_to_dict -from unstract.core.worker_models import FileExecutionResult # Import worker enums from ..enums import WebhookStatus diff --git a/workers/shared/patterns/worker_patterns.py b/workers/shared/patterns/worker_patterns.py index 77acb32ddf..a924a89162 100644 --- a/workers/shared/patterns/worker_patterns.py +++ b/workers/shared/patterns/worker_patterns.py @@ -17,9 +17,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../unstract/core/src")) -from unstract.core import ExecutionStatus, serialize_dataclass_to_dict from unstract.core.worker_models import FileExecutionResult +from unstract.core import ExecutionStatus, serialize_dataclass_to_dict + logger = logging.getLogger(__name__) diff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.py index 90009d3a55..72dca7496f 100644 --- a/workers/shared/processing/files/processor.py +++ b/workers/shared/processing/files/processor.py @@ -24,7 +24,6 @@ from typing import Any from shared.models.file_processing import FileProcessingContext - from unstract.core.data_models import ExecutionStatus, FileHashData, WorkerFileData from unstract.core.worker_models import FileProcessingResult, WorkflowExecutionResult diff --git a/workers/shared/processing/files/utils.py b/workers/shared/processing/files/utils.py index a423b7e274..0cecd09081 100644 --- a/workers/shared/processing/files/utils.py +++ b/workers/shared/processing/files/utils.py @@ -409,8 +409,7 @@ def handle_file_format_variations( else: # Single item - wrap in dict logger.warning( - f"Unexpected file data type: {type(files_data)}. " - "Wrapping as single file." + f"Unexpected file data type: {type(files_data)}. Wrapping as single file." ) return {"single_file": {"file_data": files_data}} diff --git a/workers/shared/utils/api_result_cache.py b/workers/shared/utils/api_result_cache.py index 8fcec05beb..61deefdaf9 100644 --- a/workers/shared/utils/api_result_cache.py +++ b/workers/shared/utils/api_result_cache.py @@ -9,7 +9,6 @@ from typing import Any from shared.utils.api_hub_factory import APIHubUsageUtil - from unstract.core.data_models import FileHashData from unstract.core.worker_models import ( ApiDeploymentResultStatus, diff --git a/workers/shared/utils/manual_review_factory.py b/workers/shared/utils/manual_review_factory.py index 69c1011553..7b2ff13c21 100644 --- a/workers/shared/utils/manual_review_factory.py +++ b/workers/shared/utils/manual_review_factory.py @@ -8,7 +8,6 @@ from typing import Any, Protocol from client_plugin_registry import get_client_plugin, has_client_plugin - from unstract.core.data_models import WorkerFileData from ..infrastructure.logging import WorkerLogger diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py index 7747ae5db0..6be6887821 100644 --- a/workers/shared/workflow/destination_connector.py +++ b/workers/shared/workflow/destination_connector.py @@ -34,7 +34,6 @@ ) from shared.workflow.connectors.service import WorkerConnectorService from shared.workflow.logger_helper import WorkflowLoggerHelper - from unstract.connectors.connectorkit import Connectorkit from unstract.connectors.exceptions import ConnectorError from unstract.core.data_models import ConnectionType as CoreConnectionType @@ -46,7 +45,6 @@ FileExecutionStageStatus, FileExecutionStatusTracker, ) -from unstract.filesystem import FileStorageType, FileSystem from unstract.sdk1.constants import ToolExecKey from unstract.sdk1.tool.mime_types import EXT_MIME_MAP from unstract.workflow_execution.constants import ( @@ -58,6 +56,8 @@ ExecutionFileHandler, ) +from unstract.filesystem import FileStorageType, FileSystem + if TYPE_CHECKING: from ..api_client import InternalAPIClient diff --git a/workers/shared/workflow/execution/active_file_manager.py b/workers/shared/workflow/execution/active_file_manager.py index 1814a91106..914f4172f6 100644 --- a/workers/shared/workflow/execution/active_file_manager.py +++ b/workers/shared/workflow/execution/active_file_manager.py @@ -633,7 +633,7 @@ def cleanup_cache_entries( deleted_count = cache.redis_client.delete(*batch_keys) cleaned_count += deleted_count logger_instance.debug( - f"Cleaned up {deleted_count} cache entries for {provider_uuid} (batch {i//batch_size + 1})" + f"Cleaned up {deleted_count} cache entries for {provider_uuid} (batch {i // batch_size + 1})" ) except Exception as batch_error: logger_instance.warning( diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index e38e372a91..6310f5e17c 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -1082,6 +1082,7 @@ def _copy_filesystem_file( from unstract.connectors.constants import Common from unstract.connectors.filesystems import connectors + from unstract.filesystem import FileStorageType, FileSystem # Get workflow file storage diff --git a/workers/shared/workflow/execution/tool_validation.py b/workers/shared/workflow/execution/tool_validation.py index ed40b6554c..623782fd63 100644 --- a/workers/shared/workflow/execution/tool_validation.py +++ b/workers/shared/workflow/execution/tool_validation.py @@ -9,7 +9,6 @@ from shared.api.internal_client import InternalAPIClient from shared.infrastructure.logging.workflow_logger import WorkerWorkflowLogger - from unstract.core.data_models import ExecutionStatus logger = logging.getLogger(__name__) diff --git a/workers/shared/workflow/source_connector.py b/workers/shared/workflow/source_connector.py index 950b54e462..683c7aa4bc 100644 --- a/workers/shared/workflow/source_connector.py +++ b/workers/shared/workflow/source_connector.py @@ -15,7 +15,6 @@ from shared.infrastructure.logging.logger import WorkerLogger from shared.workflow.logger_helper import WorkflowLoggerHelper - from unstract.core.data_models import ConnectionType as CoreConnectionType logger = WorkerLogger.get_logger(__name__)