update venv to activate and have same dir name in yelp and oss envs

Author: sidsatyelp

Makefile

@@ -14,6 +14,8 @@
 UID:=`id -u`
 GID:=`id -g`
 ITERATION=yelp1
+VENV_DIR=venv
+VENV_BIN=$(VENV_DIR)/bin
 
 # Determine environment (YELP or OSS)
 # Set SCL_ENV to 'YELP' if FQDN ends in '.yelpcorp.com'
@@ -24,7 +26,7 @@ else
 	SCL_ENV ?= OSS
 endif
 
-.PHONY: test tests-yelp tests-oss coverage clean venv venv-yelp venv-oss
+.PHONY: test tests-yelp tests-oss coverage clean venv
 
 
 # Main test target dispatches to environment-specific test target
@@ -35,15 +37,24 @@ else
 	$(MAKE) tests-oss
 endif
 
-tests-yelp: venv-yelp
-	tox -e py3-yelp # Assumes py3-yelp will be the tox env for yelp
+tests-yelp: venv
+	. $(VENV_BIN)/activate && $(VENV_BIN)/tox -e tests-yelp
 
-tests-oss: venv-oss
-	tox -e py3 # Assumes py3 will be the default/OSS tox env
+tests-oss: venv
+	. $(VENV_BIN)/activate && $(VENV_BIN)/tox -e tests-oss
 
 tests: test # Alias for backward compatibility or general use
 coverage: test
 
+# Main venv target, creates venv with tox
+$(VENV_BIN)/activate: setup.py tox.ini requirements-dev.txt
+	test -d $(VENV_DIR) || virtualenv -p python3.8 $(VENV_DIR)
+	$(VENV_BIN)/pip install -U pip
+	$(VENV_BIN)/pip install -U tox
+	touch $(VENV_BIN)/activate
+
+venv: $(VENV_BIN)/activate
+
 itest_%: package_%
 	docker run -h fake.docker.hostname -v $(CURDIR):/work:rw docker-dev.yelpcorp.com/$*_yelp /bin/bash -c "/work/tests/ubuntu.sh"
 	docker run -v $(CURDIR):/work:rw docker-dev.yelpcorp.com/$*_yelp chown -R $(UID):$(GID) /work
@@ -71,26 +82,12 @@ package_%:
 		'
 	docker run -v $(CURDIR):/work:rw docker-dev.yelpcorp.com/$*_yelp chown -R $(UID):$(GID) /work
 
-# Main venv target dispatches to environment-specific target
-venv:
-ifeq ($(SCL_ENV),YELP)
-	$(MAKE) venv-yelp
-else
-	$(MAKE) venv-oss
-endif
-
-venv-yelp: requirements-yelp.txt requirements-oss.txt setup.py tox.ini
-	tox -e venv-yelp # This tox env should install .[yelp]
-
-venv-oss: requirements-oss.txt setup.py tox.ini
-	tox -e venv-oss # This tox env should install normally
-
 clean:
 	rm -rf .cache
 	rm -rf dist/
 	rm -rf build/
 	rm -rf .tox
 	rm -rf service_configuration_lib.egg-info/
-	rm -rf venv
+	rm -rf $(VENV_DIR)
 	find . -name '*.pyc' -delete
 	find . -name '__pycache__' -delete

mypy.ini

@@ -5,3 +5,4 @@ warn_incomplete_stub = True
 follow_imports = silent
 ignore_missing_imports = True
 mypy_path = stubs
+strict_optional = False

service_configuration_lib/spark_config.py

@@ -1056,7 +1056,7 @@ def _handle_jira_ticket_validation(
                     try:
                         clog.config.configure(
                             scribe_host='169.254.255.254',  # Standard Scribe host
-                            scribe_port=1463,             # Standard Scribe port
+                            scribe_port='1463',             # Standard Scribe port
                             monk_disable=False,           # Ensure Monk (for clog) is enabled
                             scribe_disable=False,         # Ensure Scribe is enabled
                         )

tests/spark_config_test.py

@@ -1878,6 +1878,7 @@ def test_get_spark_conf_without_jira_ticket(self, mock_spark_srv_conf_file_with_
         assert 'ticket "None" is missing or invalid for user "regular_user"' in str(excinfo.value)
 
     @mock.patch.dict(os.environ, {'USER': 'regular_user'})
+    @mock.patch('service_configuration_lib.spark_config.clog', None)
     def test_get_spark_conf_with_jira_validation_disabled(
         self, mock_spark_srv_conf_file_with_jira_disabled, mock_log, mock_time,
     ):
@@ -1963,6 +1964,7 @@ def test_jira_ticket_check_for_different_users(
             mock_log.debug.assert_called_with('Jira ticket check not required for this job configuration.')
 
     @mock.patch.dict(os.environ, {'USER': 'regular_user'})
+    @mock.patch('service_configuration_lib.spark_config.clog', None)
     def test_get_spark_conf_jira_disabled_invalid_ticket(
         self, mock_spark_srv_conf_file_with_jira_disabled, mock_log, mock_time,
     ):

tox.ini

@@ -1,10 +1,12 @@
 [tox]
-envlist = py38, py38-yelp, venv-oss, venv-yelp
+envlist = tests-oss, tests-yelp
+skipsdist = True
 tox_pip_extensions_ext_pip_custom_platform = true
 tox_pip_extensions_ext_venv_update = true
 
 # Base environment for tests
 [testenv]
+basepython = python3.8
 deps =
     -rrequirements-dev.txt
 commands=
@@ -17,42 +19,17 @@ commands=
 usedevelop = true
 
 # OSS test environment
-[testenv:py38]
+[testenv:tests-oss]
 deps =
     {[testenv]deps}
     .
 
 # Yelp test environment
-[testenv:py38-yelp]
+[testenv:tests-yelp]
 deps =
     {[testenv]deps}
     .[yelp]
 
-# Environment for creating an OSS venv
-[testenv:venv-oss]
-basepython = /usr/bin/python3.8
-envdir = venv-oss
-deps =
-    packaging
-    more_itertools
-    jaraco.functools
-    jaraco.text
-    platformdirs==4.3.6
-    .
-commands =
-
-# Environment for creating a Yelp venv
-[testenv:venv-yelp]
-basepython = /usr/bin/python3.8
-envdir = venv-yelp
-deps =
-    packaging
-    more_itertools
-    jaraco.functools
-    jaraco.text
-    platformdirs==4.3.6
-    .[yelp]
-commands =
 
 [flake8]
 max-line-length = 119