Specify upper bound for mariadb python version (#3333)

- 1.1.0 and above causes build process to break

.github/workflows/qc_checks.yaml

@@ -77,8 +77,8 @@ jobs:
           python check_js_templates.py
       - name: Lint Javascript Files
         run: |
-          invoke render-js-files
-          npx eslint js_tmp/*.js
+          python InvenTree/manage.py prerender
+          npx eslint InvenTree/InvenTree/static_i18n/i18n/*.js
 
   html:
     name: html template files

InvenTree/InvenTree/settings.py

@@ -217,18 +217,6 @@ logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'")
 
 INSTALLED_APPS = [
 
-    # Core django modules
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'user_sessions',                # db user sessions
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-    'django.contrib.sites',
-
-    # Maintenance
-    'maintenance_mode',
-
     # InvenTree apps
     'build.apps.BuildConfig',
     'common.apps.CommonConfig',
@@ -242,6 +230,18 @@ INSTALLED_APPS = [
     'plugin.apps.PluginAppConfig',
     'InvenTree.apps.InvenTreeConfig',       # InvenTree app runs last
 
+    # Core django modules
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'user_sessions',                # db user sessions
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.sites',
+
+    # Maintenance
+    'maintenance_mode',
+
     # Third part add-ons
     'django_filters',                       # Extended filter functionality
     'rest_framework',                       # DRF (Django Rest Framework)
@@ -522,7 +522,7 @@ if "mysql" in db_engine:  # pragma: no cover
     # https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
     if "isolation_level" not in db_options:
         serializable = _is_true(
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "true")
+            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
         )
         db_options["isolation_level"] = (
             "serializable" if serializable else "read committed"

InvenTree/InvenTree/version.py

@@ -12,7 +12,7 @@ import common.models
 from InvenTree.api_version import INVENTREE_API_VERSION
 
 # InvenTree software version
-INVENTREE_SW_VERSION = "0.7.4"
+INVENTREE_SW_VERSION = "0.7.5"
 
 
 def inventreeInstanceName():

InvenTree/build/migrations/0032_auto_20211014_0632.py

@@ -24,6 +24,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         build.reference_int = ref
         build.save()
 

InvenTree/order/migrations/0052_auto_20211014_0631.py

@@ -23,6 +23,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         order.reference_int = ref
         order.save()
 
@@ -40,6 +44,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         order.reference_int = ref
         order.save()
 

InvenTree/order/test_migrations.py

@@ -56,6 +56,19 @@ class TestRefIntMigrations(MigratorTestCase):
             with self.assertRaises(AttributeError):
                 print(sales_order.reference_int)
 
+        # Create orders with very large reference values
+        self.po_pk = PurchaseOrder.objects.create(
+            supplier=supplier,
+            reference='999999999999999999999999999999999',
+            description='Big reference field',
+        ).pk
+
+        self.so_pk = SalesOrder.objects.create(
+            customer=supplier,
+            reference='999999999999999999999999999999999',
+            description='Big reference field',
+        ).pk
+
     def test_ref_field(self):
         """
         Test that the 'reference_int' field has been created and is filled out correctly
@@ -73,6 +86,15 @@ class TestRefIntMigrations(MigratorTestCase):
             self.assertEqual(po.reference_int, ii)
             self.assertEqual(so.reference_int, ii)
 
+        # Tests for orders with overly large reference values
+        po = PurchaseOrder.objects.get(pk=self.po_pk)
+        self.assertEqual(po.reference, '999999999999999999999999999999999')
+        self.assertEqual(po.reference_int, 0x7fffffff)
+
+        so = SalesOrder.objects.get(pk=self.so_pk)
+        self.assertEqual(so.reference, '999999999999999999999999999999999')
+        self.assertEqual(so.reference_int, 0x7fffffff)
+
 
 class TestShipmentMigration(MigratorTestCase):
     """

InvenTree/part/templatetags/i18n.py

@@ -0,0 +1,121 @@
+"""This module provides custom translation tags specifically for use with javascript code.
+
+Translated strings are escaped, such that they can be used as string literals in a javascript file.
+"""
+
+import django.templatetags.i18n
+from django import template
+from django.template import TemplateSyntaxError
+from django.templatetags.i18n import TranslateNode
+
+import bleach
+
+register = template.Library()
+
+
+class CustomTranslateNode(TranslateNode):
+    """Custom translation node class, which sanitizes the translated strings for javascript use"""
+
+    def render(self, context):
+        """Custom render function overrides / extends default behaviour"""
+
+        result = super().render(context)
+
+        result = bleach.clean(result)
+
+        # Remove any escape sequences
+        for seq in ['\a', '\b', '\f', '\n', '\r', '\t', '\v']:
+            result = result.replace(seq, '')
+
+        # Remove other disallowed characters
+        for c in ['\\', '`', ';', '|', '&']:
+            result = result.replace(c, '')
+
+        # Escape any quotes contained in the string
+        result = result.replace("'", r"\'")
+        result = result.replace('"', r'\"')
+
+        # Return the 'clean' resulting string
+        return result
+
+
+@register.tag("translate")
+@register.tag("trans")
+def do_translate(parser, token):
+    """Custom translation function, lifted from https://github.com/django/django/blob/main/django/templatetags/i18n.py
+
+    The only difference is that we pass this to our custom rendering node class
+    """
+
+    bits = token.split_contents()
+    if len(bits) < 2:
+        raise TemplateSyntaxError("'%s' takes at least one argument" % bits[0])
+    message_string = parser.compile_filter(bits[1])
+    remaining = bits[2:]
+
+    noop = False
+    asvar = None
+    message_context = None
+    seen = set()
+    invalid_context = {"as", "noop"}
+
+    while remaining:
+        option = remaining.pop(0)
+        if option in seen:
+            raise TemplateSyntaxError(
+                "The '%s' option was specified more than once." % option,
+            )
+        elif option == "noop":
+            noop = True
+        elif option == "context":
+            try:
+                value = remaining.pop(0)
+            except IndexError:
+                raise TemplateSyntaxError(
+                    "No argument provided to the '%s' tag for the context option."
+                    % bits[0]
+                )
+            if value in invalid_context:
+                raise TemplateSyntaxError(
+                    "Invalid argument '%s' provided to the '%s' tag for the context "
+                    "option" % (value, bits[0]),
+                )
+            message_context = parser.compile_filter(value)
+        elif option == "as":
+            try:
+                value = remaining.pop(0)
+            except IndexError:
+                raise TemplateSyntaxError(
+                    "No argument provided to the '%s' tag for the as option." % bits[0]
+                )
+            asvar = value
+        else:
+            raise TemplateSyntaxError(
+                "Unknown argument for '%s' tag: '%s'. The only options "
+                "available are 'noop', 'context' \"xxx\", and 'as VAR'."
+                % (
+                    bits[0],
+                    option,
+                )
+            )
+        seen.add(option)
+
+    return CustomTranslateNode(message_string, noop, asvar, message_context)
+
+
+# Re-register tags which we have not explicitly overridden
+register.tag("blocktrans", django.templatetags.i18n.do_block_translate)
+register.tag("blocktranslate", django.templatetags.i18n.do_block_translate)
+
+register.tag("language", django.templatetags.i18n.language)
+
+register.tag("get_available_languages", django.templatetags.i18n.do_get_available_languages)
+register.tag("get_language_info", django.templatetags.i18n.do_get_language_info)
+register.tag("get_language_info_list", django.templatetags.i18n.do_get_language_info_list)
+register.tag("get_current_language", django.templatetags.i18n.do_get_current_language)
+register.tag("get_current_language_bidi", django.templatetags.i18n.do_get_current_language_bidi)
+
+register.filter("language_name", django.templatetags.i18n.language_name)
+register.filter("language_name_translated", django.templatetags.i18n.language_name_translated)
+register.filter("language_name_local", django.templatetags.i18n.language_name_local)
+register.filter("language_bidi", django.templatetags.i18n.language_bidi)

InvenTree/plugin/base/integration/mixins.py

@@ -467,6 +467,10 @@ class APICallMixin:
         if endpoint_is_url:
             url = endpoint
         else:
+
+            if endpoint.startswith('/'):
+                endpoint = endpoint[1:]
+
             url = f'{self.api_url}/{endpoint}'
 
         # build kwargs for call
@@ -474,6 +478,7 @@ class APICallMixin:
             'url': url,
             'headers': headers,
         }
+
         if data:
             kwargs['data'] = json.dumps(data)
 

InvenTree/plugin/base/integration/test_mixins.py

@@ -1,4 +1,4 @@
-""" Unit tests for base mixins for plugins """
+"""Unit tests for base mixins for plugins."""
 
 from django.conf import settings
 from django.test import TestCase
@@ -17,7 +17,10 @@ from plugin.urls import PLUGIN_BASE
 
 
 class BaseMixinDefinition:
+    """Mixin to test the meta functions of all mixins."""
+
     def test_mixin_name(self):
+        """Test that the mixin registers itseld correctly."""
         # mixin name
         self.assertIn(self.MIXIN_NAME, [item['key'] for item in self.mixin.registered_mixins])
         # human name
@@ -25,6 +28,8 @@ class BaseMixinDefinition:
 
 
 class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
+    """Tests for SettingsMixin."""
+
     MIXIN_HUMAN_NAME = 'Settings'
     MIXIN_NAME = 'settings'
     MIXIN_ENABLE_CHECK = 'has_settings'
@@ -32,6 +37,7 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
     TEST_SETTINGS = {'SETTING1': {'default': '123', }}
 
     def setUp(self):
+        """Setup for all tests."""
         class SettingsCls(SettingsMixin, InvenTreePlugin):
             SETTINGS = self.TEST_SETTINGS
         self.mixin = SettingsCls()
@@ -43,6 +49,7 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
         super().setUp()
 
     def test_function(self):
+        """Test that the mixin functions."""
         # settings variable
         self.assertEqual(self.mixin.settings, self.TEST_SETTINGS)
 
@@ -60,11 +67,14 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
 
 
 class UrlsMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for UrlsMixin."""
+
     MIXIN_HUMAN_NAME = 'URLs'
     MIXIN_NAME = 'urls'
     MIXIN_ENABLE_CHECK = 'has_urls'
 
     def setUp(self):
+        """Setup for all tests."""
         class UrlsCls(UrlsMixin, InvenTreePlugin):
             def test():
                 return 'ccc'
@@ -76,6 +86,7 @@ class UrlsMixinTest(BaseMixinDefinition, TestCase):
         self.mixin_nothing = NoUrlsCls()
 
     def test_function(self):
+        """Test that the mixin functions."""
         plg_name = self.mixin.plugin_name()
 
         # base_url
@@ -99,26 +110,32 @@ class UrlsMixinTest(BaseMixinDefinition, TestCase):
 
 
 class AppMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for AppMixin."""
+
     MIXIN_HUMAN_NAME = 'App registration'
     MIXIN_NAME = 'app'
     MIXIN_ENABLE_CHECK = 'has_app'
 
     def setUp(self):
+        """Setup for all tests."""
         class TestCls(AppMixin, InvenTreePlugin):
             pass
         self.mixin = TestCls()
 
     def test_function(self):
-        # test that this plugin is in settings
+        """Test that the sample plugin registers in settings."""
         self.assertIn('plugin.samples.integration', settings.INSTALLED_APPS)
 
 
 class NavigationMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for NavigationMixin."""
+
     MIXIN_HUMAN_NAME = 'Navigation Links'
     MIXIN_NAME = 'navigation'
     MIXIN_ENABLE_CHECK = 'has_naviation'
 
     def setUp(self):
+        """Setup for all tests."""
         class NavigationCls(NavigationMixin, InvenTreePlugin):
             NAVIGATION = [
                 {'name': 'aa', 'link': 'plugin:test:test_view'},
@@ -131,6 +148,7 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
         self.nothing_mixin = NothingNavigationCls()
 
     def test_function(self):
+        """Test that a correct configuration functions."""
         # check right configuration
         self.assertEqual(self.mixin.navigation, [{'name': 'aa', 'link': 'plugin:test:test_view'}, ])
 
@@ -139,7 +157,7 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(self.nothing_mixin.navigation_name, '')
 
     def test_fail(self):
-        # check wrong links fails
+        """Test that wrong links fail."""
         with self.assertRaises(NotImplementedError):
             class NavigationCls(NavigationMixin, InvenTreePlugin):
                 NAVIGATION = ['aa', 'aa']
@@ -147,11 +165,15 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
 
 
 class APICallMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for APICallMixin."""
+
     MIXIN_HUMAN_NAME = 'API calls'
     MIXIN_NAME = 'api_call'
     MIXIN_ENABLE_CHECK = 'has_api_call'
 
     def setUp(self):
+        """Setup for all tests."""
+
         class MixinCls(APICallMixin, SettingsMixin, InvenTreePlugin):
             NAME = "Sample API Caller"
 
@@ -163,40 +185,47 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
                 'API_URL': {
                     'name': 'External URL',
                     'description': 'Where is your API located?',
-                    'default': 'reqres.in',
+                    'default': 'https://api.github.com',
                 },
             }
+
             API_URL_SETTING = 'API_URL'
             API_TOKEN_SETTING = 'API_TOKEN'
 
+            @property
+            def api_url(self):
+                """Override API URL for this test"""
+                return "https://api.github.com"
+
             def get_external_url(self, simple: bool = True):
-                '''
-                returns data from the sample endpoint
-                '''
-                return self.api_call('api/users/2', simple_response=simple)
+                """Returns data from the sample endpoint."""
+                return self.api_call('orgs/inventree', simple_response=simple)
+
         self.mixin = MixinCls()
 
         class WrongCLS(APICallMixin, InvenTreePlugin):
             pass
+
         self.mixin_wrong = WrongCLS()
 
         class WrongCLS2(APICallMixin, InvenTreePlugin):
             API_URL_SETTING = 'test'
+
         self.mixin_wrong2 = WrongCLS2()
 
     def test_base_setup(self):
-        """Test that the base settings work"""
+        """Test that the base settings work."""
         # check init
         self.assertTrue(self.mixin.has_api_call)
         # api_url
-        self.assertEqual('https://reqres.in', self.mixin.api_url)
+        self.assertEqual('https://api.github.com', self.mixin.api_url)
 
         # api_headers
         headers = self.mixin.api_headers
         self.assertEqual(headers, {'Bearer': '', 'Content-Type': 'application/json'})
 
     def test_args(self):
-        """Test that building up args work"""
+        """Test that building up args work."""
         # api_build_url_args
         # 1 arg
         result = self.mixin.api_build_url_args({'a': 'b'})
@@ -209,11 +238,13 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(result, '?a=b&c=d,e,f')
 
     def test_api_call(self):
-        """Test that api calls work"""
+        """Test that api calls work."""
         # api_call
         result = self.mixin.get_external_url()
         self.assertTrue(result)
-        self.assertIn('data', result,)
+
+        for key in ['login', 'email', 'name', 'twitter_username']:
+            self.assertIn(key, result)
 
         # api_call without json conversion
         result = self.mixin.get_external_url(False)
@@ -221,25 +252,25 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(result.reason, 'OK')
 
         # api_call with full url
-        result = self.mixin.api_call('https://reqres.in/api/users/2', endpoint_is_url=True)
+        result = self.mixin.api_call('https://api.github.com/orgs/inventree', endpoint_is_url=True)
         self.assertTrue(result)
 
         # api_call with post and data
         result = self.mixin.api_call(
-            'api/users/',
-            data={"name": "morpheus", "job": "leader"},
-            method='POST'
+            'repos/inventree/InvenTree',
+            method='GET'
         )
+
         self.assertTrue(result)
-        self.assertEqual(result['name'], 'morpheus')
+        self.assertEqual(result['name'], 'InvenTree')
+        self.assertEqual(result['html_url'], 'https://github.com/inventree/InvenTree')
 
         # api_call with filter
-        result = self.mixin.api_call('api/users', url_args={'page': '2'})
+        result = self.mixin.api_call('repos/inventree/InvenTree/stargazers', url_args={'page': '2'})
         self.assertTrue(result)
-        self.assertEqual(result['page'], 2)
 
     def test_function_errors(self):
-        """Test function errors"""
+        """Test function errors."""
         # wrongly defined plugins should not load
         with self.assertRaises(MixinNotImplementedError):
             self.mixin_wrong.has_api_call()
@@ -250,7 +281,7 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
 
 
 class PanelMixinTests(InvenTreeTestCase):
-    """Test that the PanelMixin plugin operates correctly"""
+    """Test that the PanelMixin plugin operates correctly."""
 
     fixtures = [
         'category',
@@ -262,8 +293,7 @@ class PanelMixinTests(InvenTreeTestCase):
     roles = 'all'
 
     def test_installed(self):
-        """Test that the sample panel plugin is installed"""
-
+        """Test that the sample panel plugin is installed."""
         plugins = registry.with_mixin('panel')
 
         self.assertTrue(len(plugins) > 0)
@@ -275,8 +305,7 @@ class PanelMixinTests(InvenTreeTestCase):
         self.assertEqual(len(plugins), 0)
 
     def test_disabled(self):
-        """Test that the panels *do not load* if the plugin is not enabled"""
-
+        """Test that the panels *do not load* if the plugin is not enabled."""
         plugin = registry.get_plugin('samplepanel')
 
         plugin.set_setting('ENABLE_HELLO_WORLD', True)
@@ -305,10 +334,7 @@ class PanelMixinTests(InvenTreeTestCase):
             self.assertNotIn('Custom Part Panel', str(response.content))
 
     def test_enabled(self):
-        """
-        Test that the panels *do* load if the plugin is enabled
-        """
-
+        """Test that the panels *do* load if the plugin is enabled."""
         plugin = registry.get_plugin('samplepanel')
 
         self.assertEqual(len(registry.with_mixin('panel', active=True)), 0)
@@ -382,8 +408,7 @@ class PanelMixinTests(InvenTreeTestCase):
         self.assertEqual(Error.objects.count(), n_errors + len(urls))
 
     def test_mixin(self):
-        """Test that ImplementationError is raised"""
-
+        """Test that ImplementationError is raised."""
         with self.assertRaises(MixinNotImplementedError):
             class Wrong(PanelMixin, InvenTreePlugin):
                 pass

InvenTree/stock/migrations/0069_auto_20211109_2347.py

@@ -28,6 +28,9 @@ def update_serials(apps, schema_editor):
             except:
                 serial = 0
 
+        # Ensure the integer value is not too large for the database field
+        if serial > 0x7fffffff:
+            serial = 0x7fffffff
 
         item.serial_int = serial
         item.save()

InvenTree/stock/test_migrations.py

@@ -0,0 +1,69 @@
+"""Unit tests for data migrations in the 'stock' app"""
+
+from django_test_migrations.contrib.unittest_case import MigratorTestCase
+
+from InvenTree import helpers
+
+
+class TestSerialNumberMigration(MigratorTestCase):
+    """Test data migration which updates serial numbers"""
+
+    migrate_from = ('stock', '0067_alter_stockitem_part')
+    migrate_to = ('stock', helpers.getNewestMigrationFile('stock'))
+
+    def prepare(self):
+        """Create initial data for this migration"""
+
+        Part = self.old_state.apps.get_model('part', 'part')
+        StockItem = self.old_state.apps.get_model('stock', 'stockitem')
+
+        # Create a base part
+        my_part = Part.objects.create(
+            name='PART-123',
+            description='Some part',
+            active=True,
+            trackable=True,
+            level=0,
+            tree_id=0,
+            lft=0, rght=0
+        )
+
+        # Create some serialized stock items
+        for sn in range(10, 20):
+            StockItem.objects.create(
+                part=my_part,
+                quantity=1,
+                serial=sn,
+                level=0,
+                tree_id=0,
+                lft=0, rght=0
+            )
+
+        # Create a stock item with a very large serial number
+        item = StockItem.objects.create(
+            part=my_part,
+            quantity=1,
+            serial='9999999999999999999999999999999999999999999999999999999999999',
+            level=0,
+            tree_id=0,
+            lft=0, rght=0
+        )
+
+        self.big_ref_pk = item.pk
+
+    def test_migrations(self):
+        """Test that the migrations have been applied correctly"""
+
+        StockItem = self.new_state.apps.get_model('stock', 'stockitem')
+
+        # Check that the serial number integer conversion has been applied correctly
+        for sn in range(10, 20):
+            item = StockItem.objects.get(serial_int=sn)
+
+            self.assertEqual(item.serial, str(sn))
+
+        big_ref_item = StockItem.objects.get(pk=self.big_ref_pk)
+
+        # Check that the StockItem maximum serial number
+        self.assertEqual(big_ref_item.serial, '9999999999999999999999999999999999999999999999999999999999999')
+        self.assertEqual(big_ref_item.serial_int, 0x7fffffff)

docker/requirements.txt

@@ -9,7 +9,7 @@ invoke>=1.4.0                   # Invoke build tool
 psycopg2>=2.9.1
 mysqlclient>=2.0.3
 pgcli>=3.1.0
-mariadb>=1.0.7
+mariadb>=1.0.7,<1.1.0
 
 # gunicorn web server
 gunicorn>=20.1.0