Bugfix for auto-backup task (#4406) (#4414)

* Bugfix for auto-backup task

- Dont' mix datetime with timezone

* Use different variable names

- If old names are stuck in the database, error could still occur

* Exclude internal settings from API

* Make INVENTREE_LATEST_VERSION setting an internal one

(cherry picked from commit fd84e590ec)

Dockerfile

@@ -64,7 +64,7 @@ RUN apt-get install -y  --no-install-recommends \
     # Weasyprint requirements : https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#debian-11
     poppler-utils libpango-1.0-0 libpangoft2-1.0-0 \
     # Image format support
-    libjpeg-dev webp \
+    libjpeg-dev webp libwebp-dev \
     # SQLite support
     sqlite3 \
     # PostgreSQL support

InvenTree/InvenTree/config.py

@@ -13,6 +13,25 @@ CONFIG_DATA = None
 CONFIG_LOOKUPS = {}
 
 
+def to_list(value, delimiter=','):
+    """Take a configuration setting and make sure it is a list.
+
+    For example, we might have a configuration setting taken from the .config file,
+    which is already a list.
+
+    However, the same setting may be specified via an environment variable,
+    using a comma delimited string!
+    """
+
+    if type(value) in [list, tuple]:
+        return value
+
+    # Otherwise, force string value
+    value = str(value)
+
+    return [x.strip() for x in value.split(delimiter)]
+
+
 def is_true(x):
     """Shortcut function to determine if a value "looks" like a boolean"""
     return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true', 'on']
@@ -101,7 +120,12 @@ def get_setting(env_var=None, config_key=None, default_value=None, typecast=None
     """
     def try_typecasting(value, source: str):
         """Attempt to typecast the value"""
-        if typecast is not None:
+
+        # Force 'list' of strings
+        if typecast is list:
+            value = to_list(value)
+
+        elif typecast is not None:
             # Try to typecast the value
             try:
                 val = typecast(value)
@@ -109,6 +133,7 @@ def get_setting(env_var=None, config_key=None, default_value=None, typecast=None
                 return val
             except Exception as error:
                 logger.error(f"Failed to typecast '{env_var}' with value '{value}' to type '{typecast}' with error {error}")
+
         set_metadata(source)
         return value
 

InvenTree/InvenTree/settings.py

@@ -103,7 +103,8 @@ MEDIA_ROOT = config.get_media_dir()
 # List of allowed hosts (default = allow all)
 ALLOWED_HOSTS = get_setting(
     config_key='allowed_hosts',
-    default_value=['*']
+    default_value=['*'],
+    typecast=list,
 )
 
 # Cross Origin Resource Sharing (CORS) options
@@ -119,7 +120,8 @@ CORS_ORIGIN_ALLOW_ALL = get_boolean_setting(
 
 CORS_ORIGIN_WHITELIST = get_setting(
     config_key='cors.whitelist',
-    default_value=[]
+    default_value=[],
+    typecast=list,
 )
 
 # Needed for the parts importer, directly impacts the maximum parts that can be uploaded
@@ -736,9 +738,11 @@ if get_boolean_setting('TEST_TRANSLATIONS', default_value=False):  # pragma: no
     django.conf.locale.LANG_INFO = LANG_INFO
 
 # Currencies available for use
-CURRENCIES = get_setting('INVENTREE_CURRENCIES', 'currencies', [
-    'AUD', 'CAD', 'CNY', 'EUR', 'GBP', 'JPY', 'NZD', 'USD',
-])
+CURRENCIES = get_setting(
+    'INVENTREE_CURRENCIES', 'currencies',
+    ['AUD', 'CAD', 'CNY', 'EUR', 'GBP', 'JPY', 'NZD', 'USD'],
+    typecast=list,
+)
 
 # Maximum number of decimal places for currency rendering
 CURRENCY_DECIMAL_PLACES = 6
@@ -746,7 +750,7 @@ CURRENCY_DECIMAL_PLACES = 6
 # Check that each provided currency is supported
 for currency in CURRENCIES:
     if currency not in moneyed.CURRENCIES:  # pragma: no cover
-        print(f"Currency code '{currency}' is not supported")
+        logger.error(f"Currency code '{currency}' is not supported")
         sys.exit(1)
 
 # Custom currency exchange backend
@@ -795,7 +799,7 @@ IMPORT_EXPORT_USE_TRANSACTIONS = True
 SITE_ID = 1
 
 # Load the allauth social backends
-SOCIAL_BACKENDS = get_setting('INVENTREE_SOCIAL_BACKENDS', 'social_backends', [])
+SOCIAL_BACKENDS = get_setting('INVENTREE_SOCIAL_BACKENDS', 'social_backends', [], typecast=list)
 
 for app in SOCIAL_BACKENDS:
     INSTALLED_APPS.append(app)  # pragma: no cover

InvenTree/InvenTree/tasks.py

@@ -377,7 +377,7 @@ def check_for_updates():
 
     # Save the version to the database
     common.models.InvenTreeSetting.set_setting(
-        'INVENTREE_LATEST_VERSION',
+        '_INVENTREE_LATEST_VERSION',
         tag,
         None
     )
@@ -440,11 +440,11 @@ def run_backup():
     time.sleep(random.randint(1, 5))
 
     # Check for records of previous backup attempts
-    last_attempt = InvenTreeSetting.get_setting('INVENTREE_BACKUP_ATTEMPT', '', cache=False)
-    last_success = InvenTreeSetting.get_setting('INVENTREE_BACKUP_SUCCESS', '', cache=False)
+    last_attempt = InvenTreeSetting.get_setting('_INVENTREE_BACKUP_ATTEMPT', '', cache=False)
+    last_success = InvenTreeSetting.get_setting('_INVENTREE_BACKUP_SUCCESS', '', cache=False)
 
     try:
-        backup_n_days = int(InvenTreeSetting.get_setting('INVENTREE_BACKUP_DAYS', 1, cache=False))
+        backup_n_days = int(InvenTreeSetting.get_setting('_INVENTREE_BACKUP_DAYS', 1, cache=False))
     except Exception:
         backup_n_days = 1
 
@@ -456,14 +456,14 @@ def run_backup():
 
     if last_attempt:
         # Do not attempt if the 'last attempt' at backup was within 12 hours
-        threshold = timezone.now() - timezone.timedelta(hours=12)
+        threshold = datetime.now() - timedelta(hours=12)
 
         if last_attempt > threshold:
             logger.info('Last backup attempt was too recent - skipping backup operation')
             return
 
     # Record the timestamp of most recent backup attempt
-    InvenTreeSetting.set_setting('INVENTREE_BACKUP_ATTEMPT', timezone.now().isoformat(), None)
+    InvenTreeSetting.set_setting('_INVENTREE_BACKUP_ATTEMPT', datetime.now().isoformat(), None)
 
     if not last_attempt:
         # If there is no record of a previous attempt, exit quickly
@@ -479,7 +479,7 @@ def run_backup():
 
     # Exit early if the backup was successful within the number of required days
     if last_success:
-        threshold = timezone.now() - timezone.timedelta(days=backup_n_days)
+        threshold = datetime.now() - timedelta(days=backup_n_days)
 
         if last_success > threshold:
             logger.info('Last successful backup was too recent - skipping backup operation')
@@ -489,7 +489,7 @@ def run_backup():
     call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False)
 
     # Record the timestamp of most recent backup success
-    InvenTreeSetting.set_setting('INVENTREE_BACKUP_SUCCESS', datetime.now().isoformat(), None)
+    InvenTreeSetting.set_setting('_INVENTREE_BACKUP_SUCCESS', datetime.now().isoformat(), None)
 
 
 def send_email(subject, body, recipients, from_email=None, html_message=None):

InvenTree/InvenTree/test_tasks.py

@@ -108,12 +108,12 @@ class InvenTreeTaskTests(TestCase):
     def test_task_check_for_updates(self):
         """Test the task check_for_updates."""
         # Check that setting should be empty
-        self.assertEqual(InvenTreeSetting.get_setting('INVENTREE_LATEST_VERSION'), '')
+        self.assertEqual(InvenTreeSetting.get_setting('_INVENTREE_LATEST_VERSION'), '')
 
         # Get new version
         InvenTree.tasks.offload_task(InvenTree.tasks.check_for_updates)
 
         # Check that setting is not empty
-        response = InvenTreeSetting.get_setting('INVENTREE_LATEST_VERSION')
+        response = InvenTreeSetting.get_setting('_INVENTREE_LATEST_VERSION')
         self.assertNotEqual(response, '')
         self.assertTrue(bool(response))

InvenTree/InvenTree/version.py

@@ -13,7 +13,7 @@ import common.models
 from InvenTree.api_version import INVENTREE_API_VERSION
 
 # InvenTree software version
-INVENTREE_SW_VERSION = "0.10.0"
+INVENTREE_SW_VERSION = "0.10.1"
 
 
 def inventreeInstanceName():
@@ -64,9 +64,9 @@ def inventreeDocsVersion():
 def isInvenTreeUpToDate():
     """Test if the InvenTree instance is "up to date" with the latest version.
 
-    A background task periodically queries GitHub for latest version, and stores it to the database as INVENTREE_LATEST_VERSION
+    A background task periodically queries GitHub for latest version, and stores it to the database as "_INVENTREE_LATEST_VERSION"
     """
-    latest = common.models.InvenTreeSetting.get_setting('INVENTREE_LATEST_VERSION', backup_value=None, create=False)
+    latest = common.models.InvenTreeSetting.get_setting('_INVENTREE_LATEST_VERSION', backup_value=None, create=False)
 
     # No record for "latest" version - we must assume we are up to date!
     if not latest:

InvenTree/common/api.py

@@ -187,7 +187,7 @@ class SettingsList(ListAPI):
 class GlobalSettingsList(SettingsList):
     """API endpoint for accessing a list of global settings objects."""
 
-    queryset = common.models.InvenTreeSetting.objects.all()
+    queryset = common.models.InvenTreeSetting.objects.exclude(key__startswith="_")
     serializer_class = common.serializers.GlobalSettingsSerializer
 
 
@@ -216,7 +216,7 @@ class GlobalSettingsDetail(RetrieveUpdateAPI):
     """
 
     lookup_field = 'key'
-    queryset = common.models.InvenTreeSetting.objects.all()
+    queryset = common.models.InvenTreeSetting.objects.exclude(key__startswith="_")
     serializer_class = common.serializers.GlobalSettingsSerializer
 
     def get_object(self):

InvenTree/common/models.py

@@ -179,6 +179,10 @@ class BaseInvenTreeSetting(models.Model):
         """
         results = cls.objects.all()
 
+        if exclude_hidden:
+            # Keys which start with an undersore are used for internal functionality
+            results = results.exclude(key__startswith='_')
+
         # Optionally filter by user
         if user is not None:
             results = results.filter(user=user)

InvenTree/part/models.py

@@ -1718,10 +1718,8 @@ class Part(InvenTreeBarcodeMixin, MetadataMixin, MPTTModel):
         Ref: https://github.com/inventree/InvenTree/pull/3986
         """
 
-        try:
-            self.pricing.schedule_for_update()
-        except (PartPricing.DoesNotExist, IntegrityError):
-            pass
+        pricing = self.pricing
+        pricing.schedule_for_update()
 
     def get_price_info(self, quantity=1, buy=True, bom=True, internal=False):
         """Return a simplified pricing string for this part.
@@ -2295,9 +2293,11 @@ class PartPricing(common.models.MetaMixin):
         """Schedule this pricing to be updated"""
 
         try:
-            self.refresh_from_db()
+            if self.pk:
+                self.refresh_from_db()
         except (PartPricing.DoesNotExist, IntegrityError):
             # Error thrown if this PartPricing instance has already been removed
+            logger.warning(f"Error refreshing PartPricing instance for part '{self.part}'")
             return
 
         # Ensure that the referenced part still exists in the database
@@ -2305,6 +2305,7 @@ class PartPricing(common.models.MetaMixin):
             p = self.part
             p.refresh_from_db()
         except IntegrityError:
+            logger.error(f"Could not update PartPricing as Part '{self.part}' does not exist")
             return
 
         if self.scheduled_for_update:
@@ -2322,6 +2323,7 @@ class PartPricing(common.models.MetaMixin):
             self.save()
         except IntegrityError:
             # An IntegrityError here likely indicates that the referenced part has already been deleted
+            logger.error(f"Could not save PartPricing for part '{self.part}' to the database")
             return
 
         import part.tasks as part_tasks

InvenTree/stock/migrations/0071_auto_20211205_1733.py

@@ -22,7 +22,13 @@ def delete_scheduled(apps, schema_editor):
 
     if items.count() > 0:
         logger.info(f"Removing {items.count()} stock items scheduled for deletion")
-        items.delete()
+
+        # Ensure any parent / child relationships are updated!
+        for item in items:
+            childs = StockItem.objects.filter(parent=item)
+            childs.update(parent=item.parent)
+
+            item.delete()
 
     Task = apps.get_model('django_q', 'schedule')
 

InvenTree/stock/test_migrations.py

@@ -67,3 +67,65 @@ class TestSerialNumberMigration(MigratorTestCase):
         # Check that the StockItem maximum serial number
         self.assertEqual(big_ref_item.serial, '9999999999999999999999999999999999999999999999999999999999999')
         self.assertEqual(big_ref_item.serial_int, 0x7fffffff)
+
+
+class TestScheduledForDeletionMigration(MigratorTestCase):
+    """Test data migration for removing 'scheduled_for_deletion' field"""
+
+    migrate_from = ('stock', '0066_stockitem_scheduled_for_deletion')
+    migrate_to = ('stock', helpers.getNewestMigrationFile('stock'))
+
+    def prepare(self):
+        """Create some initial stock items"""
+
+        Part = self.old_state.apps.get_model('part', 'part')
+        StockItem = self.old_state.apps.get_model('stock', 'stockitem')
+
+        for idx in range(5):
+            part = Part.objects.create(
+                name=f'Part_{idx}',
+                description='Just a part, nothing to see here',
+                active=True,
+                level=0, tree_id=0,
+                lft=0, rght=0,
+            )
+
+            for jj in range(5):
+                StockItem.objects.create(
+                    part=part,
+                    quantity=jj + 5,
+                    level=0, tree_id=0,
+                    lft=0, rght=0,
+                    scheduled_for_deletion=True
+                )
+
+        # For extra points, create some parent-child relationships between stock items
+        part = Part.objects.first()
+
+        item_1 = StockItem.objects.create(
+            part=part,
+            quantity=100,
+            level=0, tree_id=0,
+            lft=0, rght=0,
+            scheduled_for_deletion=True,
+        )
+
+        for _ii in range(3):
+            StockItem.objects.create(
+                part=part,
+                quantity=200,
+                level=0, tree_id=0,
+                lft=0, rght=0,
+                scheduled_for_deletion=False,
+                parent=item_1,
+            )
+
+        self.assertEqual(StockItem.objects.count(), 29)
+
+    def test_migration(self):
+        """Test that all stock items were actually removed"""
+
+        StockItem = self.new_state.apps.get_model('stock', 'stockitem')
+
+        # All the "scheduled for deletion" items have been removed
+        self.assertEqual(StockItem.objects.count(), 3)

InvenTree/templates/js/translated/part.js

@@ -1758,13 +1758,11 @@ function loadPartTable(table, url, options={}) {
         field: 'category_detail',
         title: '{% trans "Category" %}',
         formatter: function(value, row) {
-
-            var text = shortenString(row.category_detail.pathstring);
-
-            if (row.category) {
+            if (row.category && row.category_detail) {
+                var text = shortenString(row.category_detail.pathstring);
                 return withTitle(renderLink(text, `/part/category/${row.category}/`), row.category_detail.pathstring);
             } else {
-                return '{% trans "No category" %}';
+                return '<em>{% trans "No category" %}</em>';
             }
         }
     };