Fix news feed task timeout (#6250) (#6265)

Co-authored-by: Lavissa <lavissawow@gmail.com>

InvenTree/InvenTree/admin.py

@@ -19,6 +19,9 @@ class InvenTreeResource(ModelResource):
     MAX_IMPORT_ROWS = 1000
     MAX_IMPORT_COLS = 100
 
+    # List of fields which should be converted to empty strings if they are null
+    CONVERT_NULL_FIELDS = []
+
     def import_data_inner(
         self,
         dataset,
@@ -79,6 +82,13 @@ class InvenTreeResource(ModelResource):
 
         return [f for f in fields if f.column_name not in fields_to_exclude]
 
+    def before_import_row(self, row, row_number=None, **kwargs):
+        """Run custom code before importing each row"""
+
+        for field in self.CONVERT_NULL_FIELDS:
+            if field in row and row[field] is None:
+                row[field] = ''
+
 
 class CustomRateAdmin(RateAdmin):
     """Admin interface for the Rate class"""

InvenTree/InvenTree/apps.py

@@ -12,10 +12,9 @@ from django.db import transaction
 from django.db.utils import IntegrityError, OperationalError
 
 import InvenTree.conversion
+import InvenTree.ready
 import InvenTree.tasks
 from InvenTree.config import get_setting
-from InvenTree.ready import (canAppAccessDatabase, isInMainThread,
-                             isInTestMode, isPluginRegistryLoaded)
 
 logger = logging.getLogger("inventree")
 
@@ -37,17 +36,21 @@ class InvenTreeConfig(AppConfig):
         - Adding users set in the current environment
         """
         # skip loading if plugin registry is not loaded or we run in a background thread
-        if not isPluginRegistryLoaded() or not isInMainThread():
+        if not InvenTree.ready.isPluginRegistryLoaded() or not InvenTree.ready.isInMainThread():
             return
 
-        if canAppAccessDatabase() or settings.TESTING_ENV:
+        # Skip if running migrations
+        if InvenTree.ready.isRunningMigrations():
+            return
+
+        if InvenTree.ready.canAppAccessDatabase() or settings.TESTING_ENV:
 
             self.remove_obsolete_tasks()
 
             self.collect_tasks()
             self.start_background_tasks()
 
-            if not isInTestMode():  # pragma: no cover
+            if not InvenTree.ready.isInTestMode():  # pragma: no cover
                 self.update_exchange_rates()
                 # Let the background worker check for migrations
                 InvenTree.tasks.offload_task(InvenTree.tasks.check_for_migrations)
@@ -58,7 +61,7 @@ class InvenTreeConfig(AppConfig):
         # Ensure the unit registry is loaded
         InvenTree.conversion.get_unit_registry()
 
-        if canAppAccessDatabase() or settings.TESTING_ENV:
+        if InvenTree.ready.canAppAccessDatabase() or settings.TESTING_ENV:
             self.add_user_on_startup()
 
     def remove_obsolete_tasks(self):

InvenTree/InvenTree/conversion.py

@@ -62,7 +62,7 @@ def reload_unit_registry():
         pass
 
     dt = time.time() - t_start
-    logger.debug('Loaded unit registry in %s.3f s', dt)
+    logger.debug('Loaded unit registry in %.3f s', dt)
 
     return reg
 

InvenTree/InvenTree/forms.py

@@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _
 
 from allauth.account.adapter import DefaultAccountAdapter
 from allauth.account.forms import LoginForm, SignupForm, set_form_field_order
-from allauth.exceptions import ImmediateHttpResponse
+from allauth.core.exceptions import ImmediateHttpResponse
 from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
 from allauth_2fa.adapter import OTPAdapter
 from allauth_2fa.utils import user_has_valid_totp_device
@@ -24,6 +24,7 @@ from crispy_forms.layout import Field, Layout
 from dj_rest_auth.registration.serializers import RegisterSerializer
 from rest_framework import serializers
 
+import InvenTree.sso
 from common.models import InvenTreeSetting
 from InvenTree.exceptions import log_error
 
@@ -228,7 +229,7 @@ class CustomSignupForm(SignupForm):
 
 def registration_enabled():
     """Determine whether user registration is enabled."""
-    if InvenTreeSetting.get_setting('LOGIN_ENABLE_REG') or InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO_REG'):
+    if InvenTreeSetting.get_setting('LOGIN_ENABLE_REG') or InvenTree.sso.registration_enabled():
         if settings.EMAIL_HOST:
             return True
         else:
@@ -358,6 +359,13 @@ class CustomSocialAccountAdapter(CustomUrlMixin, RegistratonMixin, DefaultSocial
         # Otherwise defer to the original allauth adapter.
         return super().login(request, user)
 
+    def authentication_error(self, request, provider_id, error=None, exception=None, extra_context=None):
+        """Callback method for authentication errors."""
+
+        # Log the error to the database
+        log_error(request.path if request else 'sso')
+        logger.error("SSO error for provider '%s' - check admin error log", provider_id)
+
 
 # override dj-rest-auth
 class CustomRegisterSerializer(RegisterSerializer):

InvenTree/InvenTree/helpers.py

@@ -840,3 +840,8 @@ def inheritors(cls):
                 subcls.add(child)
                 work.append(child)
     return subcls
+
+
+def is_ajax(request):
+    """Check if the current request is an AJAX request."""
+    return request.headers.get('x-requested-with') == 'XMLHttpRequest'

InvenTree/InvenTree/ready.py

@@ -16,7 +16,11 @@ def isImportingData():
 
 def isRunningMigrations():
     """Return True if the database is currently running migrations."""
-    return 'migrate' in sys.argv or 'makemigrations' in sys.argv
+    return any((x in sys.argv for x in [
+        'migrate',
+        'makemigrations',
+        'showmigrations'
+    ]))
 
 
 def isInMainThread():

InvenTree/InvenTree/settings.py

@@ -288,6 +288,7 @@ MIDDLEWARE = CONFIG.get('middleware', [
     'InvenTree.middleware.InvenTreeRemoteUserMiddleware',       # Remote / proxy auth
     'django_otp.middleware.OTPMiddleware',                      # MFA support
     'InvenTree.middleware.CustomAllauthTwoFactorMiddleware',    # Flow control for allauth
+    'allauth.account.middleware.AccountMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'InvenTree.middleware.AuthRequiredMiddleware',
@@ -956,6 +957,13 @@ SITE_ID = 1
 SOCIAL_BACKENDS = get_setting('INVENTREE_SOCIAL_BACKENDS', 'social_backends', [], typecast=list)
 
 for app in SOCIAL_BACKENDS:
+
+    # Ensure that the app starts with 'allauth.socialaccount.providers'
+    social_prefix = 'allauth.socialaccount.providers.'
+
+    if not app.startswith(social_prefix):  # pragma: no cover
+        app = social_prefix + app
+
     INSTALLED_APPS.append(app)  # pragma: no cover
 
 SOCIALACCOUNT_PROVIDERS = get_setting('INVENTREE_SOCIAL_PROVIDERS', 'social_providers', None, typecast=dict)

InvenTree/InvenTree/social_auth_urls.py

@@ -2,13 +2,10 @@
 import logging
 from importlib import import_module
 
-from django.urls import include, path, reverse
+from django.urls import NoReverseMatch, include, path, reverse
 
 from allauth.account.models import EmailAddress
 from allauth.socialaccount import providers
-from allauth.socialaccount.models import SocialApp
-from allauth.socialaccount.providers.keycloak.views import \
-    KeycloakOAuth2Adapter
 from allauth.socialaccount.providers.oauth2.views import (OAuth2Adapter,
                                                           OAuth2LoginView)
 from drf_spectacular.utils import OpenApiResponse, extend_schema
@@ -16,6 +13,7 @@ from rest_framework.exceptions import NotFound
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 
+import InvenTree.sso
 from common.models import InvenTreeSetting
 from InvenTree.mixins import CreateAPI, ListAPI, ListCreateAPI
 from InvenTree.serializers import InvenTreeModelSerializer
@@ -51,14 +49,6 @@ def handle_oauth2(adapter: OAuth2Adapter):
     ]
 
 
-def handle_keycloak():
-    """Define urls for keycloak."""
-    return [
-        path('login/', GenericOAuth2ApiLoginView.adapter_view(KeycloakOAuth2Adapter), name='keycloak_api_login'),
-        path('connect/', GenericOAuth2ApiConnectView.adapter_view(KeycloakOAuth2Adapter), name='keycloak_api_connet'),
-    ]
-
-
 legacy = {
     'twitter': 'twitter_oauth2',
     'bitbucket': 'bitbucket_oauth2',
@@ -72,10 +62,13 @@ legacy = {
 social_auth_urlpatterns = []
 
 provider_urlpatterns = []
-for provider in providers.registry.get_list():
+
+for name, provider in providers.registry.provider_map.items():
+
     try:
         prov_mod = import_module(provider.get_package() + ".views")
     except ImportError:
+        logger.exception("Could not import authentication provider %s", name)
         continue
 
     # Try to extract the adapter class
@@ -89,8 +82,6 @@ for provider in providers.registry.get_list():
         if provider.id in legacy:
             logger.warning('`%s` is not supported on platform UI. Use `%s` instead.', provider.id, legacy[provider.id])
             continue
-        elif provider.id == 'keycloak':
-            urls = handle_keycloak()
         else:
             logger.error('Found handler that is not yet ready for platform UI: `%s`. Open an feature request on GitHub if you need it implemented.', provider.id)
             continue
@@ -107,26 +98,31 @@ class SocialProviderListView(ListAPI):
     def get(self, request, *args, **kwargs):
         """Get the list of providers."""
         provider_list = []
-        for provider in providers.registry.get_list():
+        for provider in providers.registry.provider_map.values():
             provider_data = {
                 'id': provider.id,
                 'name': provider.name,
-                'login': request.build_absolute_uri(reverse(f'{provider.id}_api_login')),
-                'connect': request.build_absolute_uri(reverse(f'{provider.id}_api_connect')),
                 'configured': False
             }
+
             try:
-                provider_app = provider.get_app(request)
-                provider_data['display_name'] = provider_app.name
-                provider_data['configured'] = True
-            except SocialApp.DoesNotExist:
-                provider_data['display_name'] = provider.name
+                provider_data['login'] = request.build_absolute_uri(reverse(f'{provider.id}_api_login'))
+            except NoReverseMatch:
+                provider_data['login'] = None
+
+            try:
+                provider_data['connect'] = request.build_absolute_uri(reverse(f'{provider.id}_api_connect'))
+            except NoReverseMatch:
+                provider_data['connect'] = None
+
+            provider_data['configured'] = InvenTree.sso.check_provider(provider)
+            provider_data['display_name'] = InvenTree.sso.provider_display_name(provider)
 
             provider_list.append(provider_data)
 
         data = {
-            'sso_enabled': InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO'),
-            'sso_registration': InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO_REG'),
+            'sso_enabled': InvenTree.sso.login_enabled(),
+            'sso_registration': InvenTree.sso.registration_enabled(),
             'mfa_required': InvenTreeSetting.get_setting('LOGIN_ENFORCE_MFA'),
             'providers': provider_list
         }

InvenTree/InvenTree/sso.py

@@ -0,0 +1,81 @@
+"""Helper functions for Single Sign On functionality"""
+
+
+import logging
+
+from common.models import InvenTreeSetting
+from InvenTree.helpers import str2bool
+
+logger = logging.getLogger('inventree')
+
+
+def get_provider_app(provider):
+    """Return the SocialApp object for the given provider"""
+
+    from allauth.socialaccount.models import SocialApp
+
+    try:
+        apps = SocialApp.objects.filter(provider__iexact=provider.id)
+    except SocialApp.DoesNotExist:
+        logger.warning("SSO SocialApp not found for provider '%s'", provider.id)
+        return None
+
+    if apps.count() > 1:
+        logger.warning("Multiple SocialApps found for provider '%s'", provider.id)
+
+    if apps.count() == 0:
+        logger.warning("SSO SocialApp not found for provider '%s'", provider.id)
+
+    return apps.first()
+
+
+def check_provider(provider, raise_error=False):
+    """Check if the given provider is correctly configured.
+
+    To be correctly configured, the following must be true:
+
+    - Provider must either have a registered SocialApp
+    - Must have at least one site enabled
+    """
+
+    import allauth.app_settings
+
+    # First, check that the provider is enabled
+    app = get_provider_app(provider)
+
+    if not app:
+        return False
+
+    if allauth.app_settings.SITES_ENABLED:
+        # At least one matching site must be specified
+        if not app.sites.exists():
+            logger.error("SocialApp %s has no sites configured", app)
+            return False
+
+    # At this point, we assume that the provider is correctly configured
+    return True
+
+
+def provider_display_name(provider):
+    """Return the 'display name' for the given provider"""
+
+    if app := get_provider_app(provider):
+        return app.name
+
+    # Fallback value if app not found
+    return provider.name
+
+
+def login_enabled() -> bool:
+    """Return True if SSO login is enabled"""
+    return str2bool(InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO'))
+
+
+def registration_enabled() -> bool:
+    """Return True if SSO registration is enabled"""
+    return str2bool(InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO_REG'))
+
+
+def auto_registration_enabled() -> bool:
+    """Return True if SSO auto-registration is enabled"""
+    return str2bool(InvenTreeSetting.get_setting('LOGIN_SIGNUP_SSO_AUTO'))

InvenTree/InvenTree/version.py

@@ -19,7 +19,7 @@ from dulwich.repo import NotGitRepository, Repo
 from .api_version import INVENTREE_API_TEXT, INVENTREE_API_VERSION
 
 # InvenTree software version
-INVENTREE_SW_VERSION = "0.13.0"
+INVENTREE_SW_VERSION = "0.13.2"
 
 # Discover git
 try:
@@ -105,7 +105,7 @@ def inventreeDocUrl():
 
 def inventreeAppUrl():
     """Return URL for InvenTree app site."""
-    return f'{inventreeDocUrl()}/app/app',
+    return f'{inventreeDocUrl()}/app/app/'
 
 
 def inventreeCreditsUrl():

InvenTree/InvenTree/views.py

@@ -33,7 +33,7 @@ from part.models import PartCategory
 from users.models import RuleSet, check_user_role
 
 from .forms import EditUserForm, SetPasswordForm
-from .helpers import remove_non_printable_characters, strip_html_tags
+from .helpers import is_ajax, remove_non_printable_characters, strip_html_tags
 
 
 def auth_request(request):
@@ -258,7 +258,7 @@ class AjaxMixin(InvenTreeRoleMixin):
         if not data:
             data = {}
 
-        if not request.is_ajax():
+        if not is_ajax(request):
             return HttpResponseRedirect('/')
 
         if context is None:

InvenTree/common/apps.py

@@ -4,7 +4,7 @@ import logging
 
 from django.apps import AppConfig
 
-from InvenTree.ready import isImportingData
+import InvenTree.ready
 
 logger = logging.getLogger('inventree')
 
@@ -19,6 +19,10 @@ class CommonConfig(AppConfig):
 
     def ready(self):
         """Initialize restart flag clearance on startup."""
+
+        if InvenTree.ready.isRunningMigrations():
+            return
+
         self.clear_restart_flag()
 
     def clear_restart_flag(self):
@@ -29,7 +33,7 @@ class CommonConfig(AppConfig):
             if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False, cache=False):
                 logger.info("Clearing SERVER_RESTART_REQUIRED flag")
 
-                if not isImportingData():
+                if not InvenTree.ready.isImportingData():
                     common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', False, None)
         except Exception:
             pass

InvenTree/common/tasks.py

@@ -10,6 +10,7 @@ from django.db.utils import IntegrityError, OperationalError
 from django.utils import timezone
 
 import feedparser
+import requests
 
 from InvenTree.helpers_model import getModelsWithMixin
 from InvenTree.models import InvenTreeNotesMixin
@@ -45,11 +46,16 @@ def update_news_feed():
         logger.info("Could not perform 'update_news_feed' - App registry not ready")
         return
 
+    # News feed isn't defined, no need to continue
+    if not settings.INVENTREE_NEWS_URL or type(settings.INVENTREE_NEWS_URL) is not str:
+        return
+
     # Fetch and parse feed
     try:
-        d = feedparser.parse(settings.INVENTREE_NEWS_URL)
-    except Exception as entry:  # pragma: no cover
-        logger.warning("update_news_feed: Error parsing the newsfeed", entry)
+        feed = requests.get(settings.INVENTREE_NEWS_URL)
+        d = feedparser.parse(feed.content)
+    except Exception:  # pragma: no cover
+        logger.warning('update_news_feed: Error parsing the newsfeed')
         return
 
     # Get a reference list

InvenTree/common/test_tasks.py

@@ -1,8 +1,9 @@
 """Tests for tasks in app common."""
 
+from django.conf import settings
 from django.test import TestCase
 
-from common.models import NotificationEntry
+from common.models import NewsFeedEntry, NotificationEntry
 from InvenTree.tasks import offload_task
 
 from . import tasks as common_tasks
@@ -15,4 +16,78 @@ class TaskTest(TestCase):
         """Test that the task `delete_old_notifications` runs through without errors."""
         # check empty run
         self.assertEqual(NotificationEntry.objects.all().count(), 0)
-        offload_task(common_tasks.delete_old_notifications,)
+        offload_task(common_tasks.delete_old_notifications)
+
+
+class NewsFeedTests(TestCase):
+    """Tests for update_news_feed task.
+
+    Tests cover different networking and addressing possibilities.
+    """
+
+    def setUp(self):
+        """Setup for tests."""
+        # Needs to be set to allow SQLite to store entries
+        settings.USE_TZ = True
+
+        # Store setting to restore on teardown
+        self.news_url = settings.INVENTREE_NEWS_URL
+
+        NewsFeedEntry.objects.all().delete()
+
+    def tearDown(self):
+        """Teardown for tests."""
+        # Restore proper setting
+        settings.INVENTREE_NEWS_URL = self.news_url
+
+        NewsFeedEntry.objects.all().delete()
+
+    def test_valid_url(self):
+        """Tests that news feed is updated when accessing a valid URL."""
+        try:
+            common_tasks.update_news_feed()
+        except Exception as ex:
+            self.fail(f'News feed raised exceptions: {ex}')
+
+        self.assertNotEqual(NewsFeedEntry.objects.all().count(), 0)
+
+    def test_connection_error(self):
+        """Test connecting to an unavailable endpoint.
+
+        This also emulates calling the endpoint behind a blocking proxy.
+        """
+        settings.INVENTREE_NEWS_URL = 'http://10.255.255.1:81'
+
+        with self.assertLogs('inventree', level='WARNING'):
+            common_tasks.update_news_feed()
+
+        self.assertEqual(NewsFeedEntry.objects.all().count(), 0)
+
+    def test_unset_url(self):
+        """Test that no call is made to news feed if URL setting is invalid."""
+        settings.INVENTREE_NEWS_URL = ''
+
+        self.assertTrue(
+            offload_task(common_tasks.update_news_feed)
+        )  # Task considered complete
+        self.assertEqual(
+            NewsFeedEntry.objects.all().count(), 0
+        )  # No Feed entries created
+
+        settings.INVENTREE_NEWS_URL = 0
+
+        self.assertTrue(
+            offload_task(common_tasks.update_news_feed)
+        )  # Task considered complete
+        self.assertEqual(
+            NewsFeedEntry.objects.all().count(), 0
+        )  # No Feed entries created
+
+        settings.INVENTREE_NEWS_URL = None
+
+        self.assertTrue(
+            offload_task(common_tasks.update_news_feed)
+        )  # Task considered complete
+        self.assertEqual(
+            NewsFeedEntry.objects.all().count(), 0
+        )  # No Feed entries created

InvenTree/company/serializers.py

@@ -337,6 +337,7 @@ class SupplierPartSerializer(InvenTreeTagModelSerializer):
         read_only_fields = [
             'availability_updated',
             'barcode_hash',
+            'pack_quantity_native',
         ]
 
     tags = TagListSerializerField(required=False)
@@ -375,6 +376,8 @@ class SupplierPartSerializer(InvenTreeTagModelSerializer):
     in_stock = serializers.FloatField(read_only=True)
     available = serializers.FloatField(required=False)
 
+    pack_quantity_native = serializers.FloatField(read_only=True)
+
     part_detail = PartBriefSerializer(source='part', many=False, read_only=True)
 
     supplier_detail = CompanyBriefSerializer(source='supplier', many=False, read_only=True)

InvenTree/config_template.yaml

@@ -233,13 +233,13 @@ remote_login_header: HTTP_REMOTE_USER
 # social_backends:
 #  - 'allauth.socialaccount.providers.google'
 #  - 'allauth.socialaccount.providers.github'
-#  - 'allauth.socialaccount.providers.keycloak'
 
 # Add specific settings for social account providers (if required)
+# Refer to the djngo-allauth documentation for more details:
+# https://docs.allauth.org/en/latest/socialaccount/provider_configuration.html
 # social_providers:
-#   keycloak:
-#     KEYCLOAK_URL: 'https://keycloak.custom/auth'
-#     KEYCLOAK_REALM: 'master'
+#   github:
+#     VERIFIED_EMAIL: true
 
 # Add LDAP support
 # ldap:

InvenTree/label/apps.py

@@ -12,8 +12,7 @@ from django.conf import settings
 from django.core.exceptions import AppRegistryNotReady
 from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 
-from InvenTree.ready import (canAppAccessDatabase, isImportingData,
-                             isInMainThread, isPluginRegistryLoaded)
+import InvenTree.ready
 
 logger = logging.getLogger("inventree")
 
@@ -37,10 +36,13 @@ class LabelConfig(AppConfig):
     def ready(self):
         """This function is called whenever the label app is loaded."""
         # skip loading if plugin registry is not loaded or we run in a background thread
-        if not isPluginRegistryLoaded() or not isInMainThread():
+        if not InvenTree.ready.isPluginRegistryLoaded() or not InvenTree.ready.isInMainThread():
             return
 
-        if canAppAccessDatabase(allow_test=False) and not isImportingData():
+        if InvenTree.ready.isRunningMigrations():
+            return
+
+        if InvenTree.ready.canAppAccessDatabase(allow_test=False) and not InvenTree.ready.isImportingData():
             try:
                 self.create_labels()  # pragma: no cover
             except (AppRegistryNotReady, IntegrityError, OperationalError, ProgrammingError):

InvenTree/order/admin.py

@@ -7,6 +7,7 @@ from import_export import widgets
 from import_export.admin import ImportExportModelAdmin
 from import_export.fields import Field
 
+import stock.models
 from InvenTree.admin import InvenTreeResource
 from order import models
 
@@ -79,9 +80,30 @@ class PurchaseOrderLineItemInlineAdmin(admin.StackedInline):
     extra = 0
 
 
+class PurchaseOrderResource(ProjectCodeResourceMixin, TotalPriceResourceMixin, InvenTreeResource):
+    """Class for managing import / export of PurchaseOrder data."""
+
+    class Meta:
+        """Metaclass"""
+        model = models.PurchaseOrder
+        skip_unchanged = True
+        clean_model_instances = True
+        exclude = [
+            'metadata',
+        ]
+
+    # Add number of line items
+    line_items = Field(attribute='line_count', widget=widgets.IntegerWidget(), readonly=True)
+
+    # Is this order overdue?
+    overdue = Field(attribute='is_overdue', widget=widgets.BooleanWidget(), readonly=True)
+
+
 class PurchaseOrderAdmin(ImportExportModelAdmin):
     """Admin class for the PurchaseOrder model"""
 
+    resource_class = PurchaseOrderResource
+
     exclude = [
         'reference_int',
     ]
@@ -107,9 +129,30 @@ class PurchaseOrderAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('supplier',)
 
 
+class SalesOrderResource(ProjectCodeResourceMixin, TotalPriceResourceMixin, InvenTreeResource):
+    """Class for managing import / export of SalesOrder data."""
+
+    class Meta:
+        """Metaclass options"""
+        model = models.SalesOrder
+        skip_unchanged = True
+        clean_model_instances = True
+        exclude = [
+            'metadata',
+        ]
+
+    # Add number of line items
+    line_items = Field(attribute='line_count', widget=widgets.IntegerWidget(), readonly=True)
+
+    # Is this order overdue?
+    overdue = Field(attribute='is_overdue', widget=widgets.BooleanWidget(), readonly=True)
+
+
 class SalesOrderAdmin(ImportExportModelAdmin):
     """Admin class for the SalesOrder model"""
 
+    resource_class = SalesOrderResource
+
     exclude = [
         'reference_int',
     ]
@@ -131,25 +174,6 @@ class SalesOrderAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('customer',)
 
 
-class PurchaseOrderResource(ProjectCodeResourceMixin, TotalPriceResourceMixin, InvenTreeResource):
-    """Class for managing import / export of PurchaseOrder data."""
-
-    class Meta:
-        """Metaclass"""
-        model = models.PurchaseOrder
-        skip_unchanged = True
-        clean_model_instances = True
-        exclude = [
-            'metadata',
-        ]
-
-    # Add number of line items
-    line_items = Field(attribute='line_count', widget=widgets.IntegerWidget(), readonly=True)
-
-    # Is this order overdue?
-    overdue = Field(attribute='is_overdue', widget=widgets.BooleanWidget(), readonly=True)
-
-
 class PurchaseOrderLineItemResource(PriceResourceMixin, InvenTreeResource):
     """Class for managing import / export of PurchaseOrderLineItem data."""
 
@@ -168,6 +192,11 @@ class PurchaseOrderLineItemResource(PriceResourceMixin, InvenTreeResource):
 
     SKU = Field(attribute='part__SKU', readonly=True)
 
+    destination = Field(
+        attribute='destination',
+        widget=widgets.ForeignKeyWidget(stock.models.StockLocation),
+    )
+
     def dehydrate_purchase_price(self, line):
         """Return a string value of the 'purchase_price' field, rather than the 'Money' object"""
         if line.purchase_price:
@@ -184,25 +213,6 @@ class PurchaseOrderExtraLineResource(PriceResourceMixin, InvenTreeResource):
         model = models.PurchaseOrderExtraLine
 
 
-class SalesOrderResource(ProjectCodeResourceMixin, TotalPriceResourceMixin, InvenTreeResource):
-    """Class for managing import / export of SalesOrder data."""
-
-    class Meta:
-        """Metaclass options"""
-        model = models.SalesOrder
-        skip_unchanged = True
-        clean_model_instances = True
-        exclude = [
-            'metadata',
-        ]
-
-    # Add number of line items
-    line_items = Field(attribute='line_count', widget=widgets.IntegerWidget(), readonly=True)
-
-    # Is this order overdue?
-    overdue = Field(attribute='is_overdue', widget=widgets.BooleanWidget(), readonly=True)
-
-
 class SalesOrderLineItemResource(PriceResourceMixin, InvenTreeResource):
     """Class for managing import / export of SalesOrderLineItem data."""
 
@@ -334,6 +344,8 @@ class ReturnOrderResource(ProjectCodeResourceMixin, TotalPriceResourceMixin, Inv
 class ReturnOrderAdmin(ImportExportModelAdmin):
     """Admin class for the ReturnOrder model"""
 
+    resource_class = ReturnOrderResource
+
     exclude = [
         'reference_int',
     ]

InvenTree/order/models.py

@@ -1926,6 +1926,7 @@ class ReturnOrder(TotalPriceMixin, Order):
         stock_item.customer = None
         stock_item.sales_order = None
         stock_item.save(add_note=False)
+        stock_item.clearAllocations()
 
         # Add a tracking entry to the StockItem
         stock_item.add_tracking_entry(

InvenTree/order/serializers.py

@@ -247,7 +247,7 @@ class PurchaseOrderCancelSerializer(serializers.Serializer):
         self.order = self.context['order']
 
         return {
-            'can_cancel': self.order.can_cancel(),
+            'can_cancel': self.order.can_cancel,
         }
 
     def save(self):
@@ -1219,7 +1219,7 @@ class SalesOrderCancelSerializer(serializers.Serializer):
         order = self.context['order']
 
         return {
-            'can_cancel': order.can_cancel(),
+            'can_cancel': order.can_cancel,
         }
 
     def save(self):

InvenTree/part/admin.py

@@ -356,9 +356,32 @@ class BomItemAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('part', 'sub_part',)
 
 
+class ParameterTemplateResource(InvenTreeResource):
+    """Class for managing ParameterTemplate import/export"""
+
+    # The following fields will be converted from None to ''
+    CONVERT_NULL_FIELDS = [
+        'choices',
+        'units'
+    ]
+
+    class Meta:
+        """Metaclass definition"""
+        model = models.PartParameterTemplate
+        skip_unchanged = True
+        report_skipped = False
+        clean_model_instances = True
+
+        exclude = [
+            'metadata',
+        ]
+
+
 class ParameterTemplateAdmin(ImportExportModelAdmin):
     """Admin class for the PartParameterTemplate model"""
 
+    resource_class = ParameterTemplateResource
+
     list_display = ('name', 'units')
 
     search_fields = ('name', 'units')

InvenTree/part/api.py

@@ -22,8 +22,8 @@ from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
 from InvenTree.filters import (ORDER_FILTER, SEARCH_ORDER_FILTER,
                                SEARCH_ORDER_FILTER_ALIAS, InvenTreeDateFilter,
                                InvenTreeSearchFilter)
-from InvenTree.helpers import (DownloadFile, increment_serial_number, isNull,
-                               str2bool, str2int)
+from InvenTree.helpers import (DownloadFile, increment_serial_number, is_ajax,
+                               isNull, str2bool, str2int)
 from InvenTree.mixins import (CreateAPI, CustomRetrieveUpdateDestroyAPI,
                               ListAPI, ListCreateAPI, RetrieveAPI,
                               RetrieveUpdateAPI, RetrieveUpdateDestroyAPI,
@@ -1069,7 +1069,7 @@ class PartList(PartMixin, APIDownloadMixin, ListCreateAPI):
         """
         if page is not None:
             return self.get_paginated_response(data)
-        elif request.is_ajax():
+        elif is_ajax(request):
             return JsonResponse(data, safe=False)
         return Response(data)
 
@@ -1740,7 +1740,7 @@ class BomList(BomMixin, ListCreateDestroyAPIView):
         """
         if page is not None:
             return self.get_paginated_response(data)
-        elif request.is_ajax():
+        elif is_ajax(request):
             return JsonResponse(data, safe=False)
         return Response(data)
 

InvenTree/part/apps.py

@@ -5,8 +5,7 @@ import logging
 from django.apps import AppConfig
 from django.db.utils import OperationalError, ProgrammingError
 
-from InvenTree.ready import (canAppAccessDatabase, isImportingData,
-                             isInMainThread, isPluginRegistryLoaded)
+import InvenTree.ready
 
 logger = logging.getLogger("inventree")
 
@@ -18,10 +17,13 @@ class PartConfig(AppConfig):
     def ready(self):
         """This function is called whenever the Part app is loaded."""
         # skip loading if plugin registry is not loaded or we run in a background thread
-        if not isPluginRegistryLoaded() or not isInMainThread():
+        if not InvenTree.ready.isPluginRegistryLoaded() or not InvenTree.ready.isInMainThread():
             return
 
-        if canAppAccessDatabase():
+        if InvenTree.ready.isRunningMigrations():
+            return
+
+        if InvenTree.ready.canAppAccessDatabase():
             self.update_trackable_status()
             self.reset_part_pricing_flags()
 
@@ -51,7 +53,7 @@ class PartConfig(AppConfig):
         """
         from .models import PartPricing
 
-        if isImportingData():
+        if InvenTree.ready.isImportingData():
             return
 
         try:

InvenTree/part/models.py

@@ -3346,7 +3346,10 @@ class PartParameterTemplate(MetadataMixin, models.Model):
                 })
 
         # Check that 'choices' are in fact valid
-        self.choices = self.choices.strip()
+        if self.choices is None:
+            self.choices = ''
+        else:
+            self.choices = str(self.choices).strip()
 
         if self.choices:
             choice_set = set()

InvenTree/part/templatetags/inventree_extras.py

@@ -539,8 +539,11 @@ def authorized_owners(group):
 @register.simple_tag()
 def object_link(url_name, pk, ref):
     """Return highlighted link to object."""
-    ref_url = reverse(url_name, kwargs={'pk': pk})
-    return mark_safe(f'<b><a href="{ref_url}">{ref}</a></b>')
+    try:
+        ref_url = reverse(url_name, kwargs={'pk': pk})
+        return mark_safe(f'<b><a href="{ref_url}">{ref}</a></b>')
+    except NoReverseMatch:
+        return None
 
 
 @register.simple_tag()

InvenTree/part/templatetags/sso.py

@@ -1,58 +1,32 @@
 """This module provides template tags pertaining to SSO functionality"""
 
-import logging
-
 from django import template
 
-from common.models import InvenTreeSetting
-from InvenTree.helpers import str2bool
+import InvenTree.sso
 
 register = template.Library()
-logger = logging.getLogger('inventree')
 
 
 @register.simple_tag()
 def sso_login_enabled():
     """Return True if single-sign-on is enabled"""
-    return str2bool(InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO'))
+    return InvenTree.sso.login_enabled()
 
 
 @register.simple_tag()
 def sso_reg_enabled():
     """Return True if single-sign-on is enabled for self-registration"""
-    return str2bool(InvenTreeSetting.get_setting('LOGIN_ENABLE_SSO_REG'))
+    return InvenTree.sso.registration_enabled()
 
 
 @register.simple_tag()
 def sso_auto_enabled():
     """Return True if single-sign-on is enabled for auto-registration"""
-    return str2bool(InvenTreeSetting.get_setting('LOGIN_SIGNUP_SSO_AUTO'))
+    return InvenTree.sso.auto_registration_enabled()
 
 
 @register.simple_tag()
 def sso_check_provider(provider):
     """Return True if the given provider is correctly configured"""
-    import allauth.app_settings
-    from allauth.socialaccount.models import SocialApp
 
-    # First, check that the provider is enabled
-    apps = SocialApp.objects.filter(provider__iexact=provider.id)
-
-    if not apps.exists():
-        logging.error(
-            "SSO SocialApp %s does not exist (known providers: %s)",
-            provider.id, [obj.provider for obj in SocialApp.objects.all()]
-        )
-        return False
-
-    # Next, check that the provider is correctly configured
-    app = apps.first()
-
-    if allauth.app_settings.SITES_ENABLED:
-        # At least one matching site must be specified
-        if not app.sites.exists():
-            logger.error("SocialApp %s has no sites configured", app)
-            return False
-
-    # At this point, we assume that the provider is correctly configured
-    return True
+    return InvenTree.sso.check_provider(provider)

InvenTree/plugin/base/barcodes/api.py

@@ -147,16 +147,16 @@ class BarcodeAssign(BarcodeView):
         """
 
         # Here we only check against 'InvenTree' plugins
-        plugins = registry.with_mixin('barcode', builtin=True)
+        inventree_barcode_plugin = registry.get_plugin('inventreebarcode')
 
         # First check if the provided barcode matches an existing database entry
-        for plugin in plugins:
-            result = plugin.scan(barcode)
+        if inventree_barcode_plugin:
+            result = inventree_barcode_plugin.scan(barcode)
 
             if result is not None:
-                result["error"] = _("Barcode matches existing item")
-                result["plugin"] = plugin.name
-                result["barcode_data"] = barcode
+                result['error'] = _('Barcode matches existing item')
+                result['plugin'] = inventree_barcode_plugin.name
+                result['barcode_data'] = barcode
 
                 raise ValidationError(result)
 

InvenTree/plugin/base/barcodes/mixins.py

@@ -6,7 +6,7 @@ import logging
 from decimal import Decimal, InvalidOperation
 
 from django.contrib.auth.models import User
-from django.db.models import F
+from django.db.models import F, Q
 from django.utils.translation import gettext_lazy as _
 
 from company.models import Company, SupplierPart
@@ -347,12 +347,17 @@ class SupplierBarcodeMixin(BarcodeMixin):
         if supplier:
             orders = orders.filter(supplier=supplier)
 
-        if customer_order_number:
-            orders = orders.filter(reference__iexact=customer_order_number)
-        elif supplier_order_number:
-            orders = orders.filter(supplier_reference__iexact=supplier_order_number)
+        # this works because reference and supplier_reference are not nullable, so if
+        # customer_order_number or supplier_order_number is None, the query won't return anything
+        reference_filter = Q(reference__iexact=customer_order_number)
+        supplier_reference_filter = Q(supplier_reference__iexact=supplier_order_number)
 
-        return orders
+        orders_union = orders.filter(reference_filter | supplier_reference_filter)
+        if orders_union.count() == 1:
+            return orders_union
+        else:
+            orders_intersection = orders.filter(reference_filter & supplier_reference_filter)
+            return orders_intersection if orders_intersection else orders_union
 
     @staticmethod
     def get_supplier_parts(sku: str = None, supplier: Company = None, mpn: str = None):

InvenTree/plugin/base/barcodes/test_barcode.py

@@ -8,6 +8,8 @@ from InvenTree.unit_test import InvenTreeAPITestCase
 from part.models import Part
 from stock.models import StockItem
 
+from .mixins import SupplierBarcodeMixin
+
 
 class BarcodeAPITest(InvenTreeAPITestCase):
     """Tests for barcode api."""
@@ -431,3 +433,142 @@ class SOAllocateTest(InvenTreeAPITestCase):
         self.line_item.refresh_from_db()
         self.assertEqual(self.line_item.allocated_quantity(), 10)
         self.assertTrue(self.line_item.is_fully_allocated())
+
+
+class SupplierBarcodeMixinTest(InvenTreeAPITestCase):
+    """Unit tests for the SupplierBarcodeMixin class."""
+
+    @classmethod
+    def setUpTestData(cls):
+        """Setup for all tests."""
+        super().setUpTestData()
+
+        cls.supplier = company.models.Company.objects.create(
+            name='Supplier Barcode Mixin Test Company', is_supplier=True
+        )
+
+        cls.supplier_other = company.models.Company.objects.create(
+            name='Other Supplier Barcode Mixin Test Company', is_supplier=True
+        )
+
+        cls.supplier_no_orders = company.models.Company.objects.create(
+            name='Supplier Barcode Mixin Test Company with no Orders', is_supplier=True
+        )
+
+        cls.purchase_order_pending = order.models.PurchaseOrder.objects.create(
+            status=order.models.PurchaseOrderStatus.PENDING.value,
+            supplier=cls.supplier,
+            supplier_reference='ORDER#1337',
+        )
+
+        cls.purchase_order_1 = order.models.PurchaseOrder.objects.create(
+            status=order.models.PurchaseOrderStatus.PLACED.value,
+            supplier=cls.supplier,
+            supplier_reference='ORDER#1338',
+        )
+
+        cls.purchase_order_duplicate_1 = order.models.PurchaseOrder.objects.create(
+            status=order.models.PurchaseOrderStatus.PLACED.value,
+            supplier=cls.supplier,
+            supplier_reference='ORDER#1339',
+        )
+
+        cls.purchase_order_duplicate_2 = order.models.PurchaseOrder.objects.create(
+            status=order.models.PurchaseOrderStatus.PLACED.value,
+            supplier=cls.supplier_other,
+            supplier_reference='ORDER#1339',
+        )
+
+    def setUp(self):
+        """Setup method for each test."""
+        super().setUp()
+
+    def test_order_not_placed(self):
+        """Check that purchase order which has not been placed doesn't get returned."""
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_pending.reference, None
+        )
+        self.assertIsNone(purchase_orders.first())
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            None, self.purchase_order_pending.supplier_reference
+        )
+        self.assertIsNone(purchase_orders.first())
+
+    def test_order_simple(self):
+        """Check that we can get a purchase order by either reference, supplier_reference or both."""
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_1.reference, None
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_1)
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            None, self.purchase_order_1.supplier_reference
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_1)
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_1.reference, self.purchase_order_1.supplier_reference
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_1)
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_1.reference,
+            self.purchase_order_1.supplier_reference,
+            supplier=self.supplier,
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_1)
+
+    def test_wrong_supplier_order(self):
+        """Check that no orders get returned if the wrong supplier is specified."""
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_1.reference, None, supplier=self.supplier_no_orders
+        )
+        self.assertIsNone(purchase_orders.first())
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            None,
+            self.purchase_order_1.supplier_reference,
+            supplier=self.supplier_no_orders,
+        )
+        self.assertIsNone(purchase_orders.first())
+
+    def test_supplier_order_duplicate(self):
+        """Test getting purchase_orders with the same supplier_reference works correctly."""
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            None, self.purchase_order_duplicate_1.supplier_reference
+        )
+        self.assertEqual(purchase_orders.count(), 2)
+        self.assertEqual(
+            set(purchase_orders),
+            {self.purchase_order_duplicate_1, self.purchase_order_duplicate_2},
+        )
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_duplicate_1.reference,
+            self.purchase_order_duplicate_1.supplier_reference,
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_duplicate_1)
+
+        # check that mixing the reference and supplier_reference doesn't work
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            self.purchase_order_duplicate_1.supplier_reference,
+            self.purchase_order_duplicate_1.reference,
+        )
+        self.assertIsNone(purchase_orders.first())
+
+        # check that specifying the right supplier works
+
+        purchase_orders = SupplierBarcodeMixin.get_purchase_orders(
+            None,
+            self.purchase_order_duplicate_1.supplier_reference,
+            supplier=self.supplier_other,
+        )
+        self.assertEqual(purchase_orders.count(), 1)
+        self.assertEqual(purchase_orders.first(), self.purchase_order_duplicate_2)

InvenTree/plugin/builtin/suppliers/lcsc.py

@@ -36,7 +36,7 @@ class LCSCPlugin(SupplierBarcodeMixin, SettingsMixin, InvenTreePlugin):
         "pm": SupplierBarcodeMixin.MANUFACTURER_PART_NUMBER,
         "pc": SupplierBarcodeMixin.SUPPLIER_PART_NUMBER,
         "qty": SupplierBarcodeMixin.QUANTITY,
-        "on": SupplierBarcodeMixin.CUSTOMER_ORDER_NUMBER,
+        "on": SupplierBarcodeMixin.SUPPLIER_ORDER_NUMBER,
     }
 
     def extract_barcode_fields(self, barcode_data: str) -> dict[str, str]:

InvenTree/plugin/builtin/suppliers/mouser.py

@@ -30,4 +30,11 @@ class MouserPlugin(SupplierBarcodeMixin, SettingsMixin, InvenTreePlugin):
     def extract_barcode_fields(self, barcode_data: str) -> dict[str, str]:
         """Get supplier_part and barcode_fields from Mouser DataMatrix-Code."""
 
-        return self.parse_ecia_barcode2d(barcode_data)
+        barcode_fields = self.parse_ecia_barcode2d(barcode_data)
+
+        # Mouser uses the custom order number ('K') field of the 2D barcode for both,
+        # the order number and the customer order number
+        if order_number := barcode_fields.get(self.CUSTOMER_ORDER_NUMBER):
+            barcode_fields.setdefault(self.SUPPLIER_ORDER_NUMBER, order_number)
+
+        return barcode_fields

InvenTree/plugin/builtin/suppliers/tme.py

@@ -35,7 +35,8 @@ class TMEPlugin(SupplierBarcodeMixin, SettingsMixin, InvenTreePlugin):
     # Custom field mapping
     TME_QRCODE_FIELDS = {
         "PN": SupplierBarcodeMixin.SUPPLIER_PART_NUMBER,
-        "PO": SupplierBarcodeMixin.CUSTOMER_ORDER_NUMBER,
+        "CPO": SupplierBarcodeMixin.CUSTOMER_ORDER_NUMBER,
+        "PO": SupplierBarcodeMixin.SUPPLIER_ORDER_NUMBER,
         "MPN": SupplierBarcodeMixin.MANUFACTURER_PART_NUMBER,
         "QTY": SupplierBarcodeMixin.QUANTITY,
     }

InvenTree/report/api.py

@@ -528,7 +528,7 @@ report_api_urls = [
         path(r'<int:pk>/', include([
             re_path(r'print/?', BuildReportPrint.as_view(), name='api-build-report-print'),
             re_path(r'metadata/', MetadataView.as_view(), {'model': BuildReport}, name='api-build-report-metadata'),
-            re_path(r'^.$', BuildReportDetail.as_view(), name='api-build-report-detail'),
+            re_path(r'^.*$', BuildReportDetail.as_view(), name='api-build-report-detail'),
         ])),
 
         # List view

InvenTree/report/apps.py

@@ -20,11 +20,13 @@ class ReportConfig(AppConfig):
 
     def ready(self):
         """This function is called whenever the report app is loaded."""
-        from InvenTree.ready import (canAppAccessDatabase, isImportingData,
-                                     isInMainThread, isPluginRegistryLoaded)
+        import InvenTree.ready
 
         # skip loading if plugin registry is not loaded or we run in a background thread
-        if not isPluginRegistryLoaded() or not isInMainThread():
+        if not InvenTree.ready.isPluginRegistryLoaded() or not InvenTree.ready.isInMainThread():
+            return
+
+        if InvenTree.ready.isRunningMigrations():
             return
 
         # Configure logging for PDF generation (disable "info" messages)
@@ -32,7 +34,7 @@ class ReportConfig(AppConfig):
         logging.getLogger('weasyprint').setLevel(logging.WARNING)
 
         # Create entries for default report templates
-        if canAppAccessDatabase(allow_test=False) and not isImportingData():
+        if InvenTree.ready.canAppAccessDatabase(allow_test=False) and not InvenTree.ready.isImportingData():
 
             try:
                 self.create_default_test_reports()

InvenTree/stock/api.py

@@ -27,8 +27,8 @@ from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
                            ListCreateDestroyAPIView, MetadataView)
 from InvenTree.filters import (ORDER_FILTER, SEARCH_ORDER_FILTER,
                                SEARCH_ORDER_FILTER_ALIAS, InvenTreeDateFilter)
-from InvenTree.helpers import (DownloadFile, extract_serial_numbers, isNull,
-                               str2bool, str2int)
+from InvenTree.helpers import (DownloadFile, extract_serial_numbers, is_ajax,
+                               isNull, str2bool, str2int)
 from InvenTree.mixins import (CreateAPI, CustomRetrieveUpdateDestroyAPI,
                               ListAPI, ListCreateAPI, RetrieveAPI,
                               RetrieveUpdateDestroyAPI)
@@ -963,7 +963,7 @@ class StockList(APIDownloadMixin, ListCreateDestroyAPIView):
 
         if page is not None:
             return self.get_paginated_response(data)
-        elif request.is_ajax():
+        elif is_ajax(request):
             return JsonResponse(data, safe=False)
         return Response(data)
 
@@ -1346,7 +1346,7 @@ class StockTrackingList(ListAPI):
 
         if page is not None:
             return self.get_paginated_response(data)
-        if request.is_ajax():
+        if is_ajax(request):
             return JsonResponse(data, safe=False)
         return Response(data)
 

InvenTree/templates/js/translated/part.js

@@ -1812,12 +1812,13 @@ function loadPartPurchaseOrderTable(table, part_id, options={}) {
                 formatter: function(value, row) {
                     let data = value;
 
-                    if (row.supplier_part_detail.pack_quantity_native != 1.0) {
-                        let total = value * row.supplier_part_detail.pack_quantity_native;
+                    if (row.supplier_part_detail && row.supplier_part_detail.pack_quantity_native != 1.0) {
+                        let pq = row.supplier_part_detail.pack_quantity_native;
+                        let total = value * pq;
 
                         data += makeIconBadge(
                             'fa-info-circle icon-blue',
-                            `{% trans "Pack Quantity" %}: ${row.pack_quantity} - {% trans "Total Quantity" %}: ${total}`
+                            `{% trans "Pack Quantity" %}: ${pq} - {% trans "Total Quantity" %}: ${total}`
                         );
                     }
 
@@ -1870,9 +1871,10 @@ function loadPartPurchaseOrderTable(table, part_id, options={}) {
                 formatter: function(value, row) {
                     var data = value;
 
-                    if (value > 0 && row.supplier_part_detail.pack_quantity_native != 1.0) {
-                        let total = value * row.supplier_part_detail.pack_quantity_native;
-                        data += `<span class='fas fa-info-circle icon-blue float-right' title='{% trans "Pack Quantity" %}: ${row.pack_quantity} - {% trans "Total Quantity" %}: ${total}'></span>`;
+                    if (value > 0 && row.supplier_part_detail && row.supplier_part_detail.pack_quantity_native != 1.0) {
+                        let pq = row.supplier_part_detail.pack_quantity_native;
+                        let total = value * pq;
+                        data += `<span class='fas fa-info-circle icon-blue float-right' title='{% trans "Pack Quantity" %}: ${pq} - {% trans "Total Quantity" %}: ${total}'></span>`;
                     }
 
                     return data;

InvenTree/templates/socialaccount/authentication_error.html

@@ -1,4 +1,4 @@
-{% extends "socialaccount/base.html" %}
+{% extends "account/base.html" %}
 
 {% load i18n %}
 

InvenTree/templates/socialaccount/login.html

@@ -1,4 +1,4 @@
-{% extends "socialaccount/base.html" %}
+{% extends "account/base.html" %}
 {% load i18n %}
 {% load sso %}
 

InvenTree/templates/socialaccount/signup.html

@@ -1,4 +1,4 @@
-{% extends "socialaccount/base.html" %}
+{% extends "account/base.html" %}
 
 {% load i18n crispy_forms_tags inventree_extras %}
 

InvenTree/users/apps.py

@@ -5,8 +5,7 @@ import logging
 from django.apps import AppConfig
 from django.db.utils import OperationalError, ProgrammingError
 
-from InvenTree.ready import (canAppAccessDatabase, isInMainThread,
-                             isPluginRegistryLoaded)
+import InvenTree.ready
 
 logger = logging.getLogger('inventree')
 
@@ -19,10 +18,14 @@ class UsersConfig(AppConfig):
     def ready(self):
         """Called when the 'users' app is loaded at runtime"""
         # skip loading if plugin registry is not loaded or we run in a background thread
-        if not isPluginRegistryLoaded() or not isInMainThread():
+        if not InvenTree.ready.isPluginRegistryLoaded() or not InvenTree.ready.isInMainThread():
             return
 
-        if canAppAccessDatabase(allow_test=True):
+        # Skip if running migrations
+        if InvenTree.ready.isRunningMigrations():
+            return
+
+        if InvenTree.ready.canAppAccessDatabase(allow_test=True):
 
             try:
                 self.assign_permissions()

contrib/packager.io/functions.sh

@@ -236,7 +236,7 @@ function update_or_install() {
   # Run update as app user
   echo "# Updating InvenTree"
   sudo -u ${APP_USER} --preserve-env=$SETUP_ENVS bash -c "cd ${APP_HOME} && pip install wheel"
-  sudo -u ${APP_USER} --preserve-env=$SETUP_ENVS bash -c "cd ${APP_HOME} && invoke update --no-frontend | sed -e 's/^/# inv update| /;'"
+  sudo -u ${APP_USER} --preserve-env=$SETUP_ENVS bash -c "cd ${APP_HOME} && invoke update | sed -e 's/^/# inv update| /;'"
 
   # Make sure permissions are correct again
   echo "# Set permissions for data dir and media: ${DATA_DIR}"

docs/docs/settings/SSO.md

@@ -9,6 +9,9 @@ InvenTree provides the possibility to use 3rd party services to authenticate use
 !!! tip "Provider Documentation"
     There are a lot of technical considerations when configuring a particular SSO provider. A good starting point is the [django-allauth documentation](https://django-allauth.readthedocs.io/en/latest/socialaccount/providers/index.html)
 
+!!! warning "Advanced Users"
+    The SSO functionality provided by django-allauth is powerful, but can prove challenging to configure. Please ensure that you understand the implications of enabling SSO for your InvenTree instance. Specific technical details of each available SSO provider are beyond the scope of this documentation - please refer to the [django-allauth documentation](https://django-allauth.readthedocs.io/en/latest/socialaccount/providers/index.html) for more information.
+
 ## SSO Configuration
 
 The basic requirements for configuring SSO are outlined below:
@@ -131,3 +134,7 @@ Make sure all users with admin privileges have sufficient passwords - they can r
 
 !!! warning "It's a secret!"
     Never share the secret key associated with your InvenTree install!
+
+## Error Handling
+
+If you encounter an error during the SSO process, the error should be logged in the InvenTree database. You can view the [error log](./logs.md) in the [admin interface](./admin.md) to see the details of the error.

requirements.in

@@ -2,7 +2,7 @@
 Django>=3.2.14,<4                       # Django package
 coreapi                                 # API documentation for djangorestframework
 cryptography>=40.0.0,!=40.0.2           # Core cryptographic functionality
-django-allauth<0.55.0                   # SSO for external providers via OpenID  # FIXED 2023-09-06 due to https://github.com/iMerica/dj-rest-auth/issues/534
+django-allauth                          # SSO for external providers via OpenID
 django-allauth-2fa                      # MFA / 2FA
 django-cleanup                          # Automated deletion of old / unused uploaded files
 django-cors-headers                     # CORS headers extension for DRF

requirements.txt

@@ -45,7 +45,7 @@ defusedxml==0.7.1
     #   python3-openid
 diff-match-patch==20230430
     # via django-import-export
-dj-rest-auth==5.0.1
+dj-rest-auth==5.0.2
     # via -r requirements.in
 django==3.2.23
     # via
@@ -81,7 +81,7 @@ django==3.2.23
     #   djangorestframework
     #   djangorestframework-simplejwt
     #   drf-spectacular
-django-allauth==0.54.0
+django-allauth==0.59.0
     # via
     #   -r requirements.in
     #   django-allauth-2fa

tasks.py

@@ -18,11 +18,10 @@ def checkPythonVersion():
 
     If the python version is not sufficient, exits with a non-zero exit code.
     """
-
     REQ_MAJOR = 3
     REQ_MINOR = 9
 
-    version = sys.version.split(" ")[0]
+    version = sys.version.split(' ')[0]
 
     valid = True
 
@@ -33,8 +32,8 @@ def checkPythonVersion():
         valid = False
 
     if not valid:
-        print(f"The installed python version ({version}) is not supported!")
-        print(f"InvenTree requires Python {REQ_MAJOR}.{REQ_MINOR} or above")
+        print(f'The installed python version ({version}) is not supported!')
+        print(f'InvenTree requires Python {REQ_MAJOR}.{REQ_MINOR} or above')
         sys.exit(1)
 
 
@@ -59,29 +58,57 @@ def apps():
     ]
 
 
-def content_excludes():
-    """Returns a list of content types to exclude from import/export."""
+def content_excludes(
+    allow_auth: bool = True,
+    allow_tokens: bool = True,
+    allow_plugins: bool = True,
+    allow_sso: bool = True,
+):
+    """Returns a list of content types to exclude from import/export.
+
+    Arguments:
+        allow_auth (bool): Allow user/group information to be exported/imported
+        allow_tokens (bool): Allow tokens to be exported/importe
+        allow_plugins (bool): Allow plugin information to be exported/imported
+        allow_sso (bool): Allow SSO tokens to be exported/imported
+    """
     excludes = [
-        "contenttypes",
-        "auth.permission",
-        "users.apitoken",
-        "error_report.error",
-        "admin.logentry",
-        "django_q.schedule",
-        "django_q.task",
-        "django_q.ormq",
-        "users.owner",
-        "exchange.rate",
-        "exchange.exchangebackend",
-        "common.notificationentry",
-        "common.notificationmessage",
-        "user_sessions.session",
+        'contenttypes',
+        'auth.permission',
+        'error_report.error',
+        'admin.logentry',
+        'django_q.schedule',
+        'django_q.task',
+        'django_q.ormq',
+        'exchange.rate',
+        'exchange.exchangebackend',
+        'common.notificationentry',
+        'common.notificationmessage',
+        'user_sessions.session',
     ]
 
-    output = ""
+    # Optionally exclude user auth data
+    if not allow_auth:
+        excludes.append('auth.group')
+        excludes.append('auth.user')
+
+    # Optionally exclude user token information
+    if not allow_tokens:
+        excludes.append('users.apitoken')
+
+    # Optionally exclude plugin information
+    if not allow_plugins:
+        excludes.append('plugin.pluginconfig')
+        excludes.append('plugin.pluginsetting')
+
+    # Optionally exclude SSO application information
+    if not allow_sso:
+        excludes.append('socialaccount.socialapp')
+
+    output = ''
 
     for e in excludes:
-        output += f"--exclude {e} "
+        output += f'--exclude {e} '
 
     return output
 
@@ -113,10 +140,10 @@ def manage(c, cmd, pty: bool = False):
         cmd: Django command to run.
         pty (bool, optional): Run an interactive session. Defaults to False.
     """
-    c.run('cd "{path}" && python3 manage.py {cmd}'.format(
-        path=managePyDir(),
-        cmd=cmd
-    ), pty=pty)
+    c.run(
+        'cd "{path}" && python3 manage.py {cmd}'.format(path=managePyDir(), cmd=cmd),
+        pty=pty,
+    )
 
 
 def yarn(c, cmd, pty: bool = False):
@@ -133,6 +160,7 @@ def yarn(c, cmd, pty: bool = False):
 
 def node_available(versions: bool = False, bypass_yarn: bool = False):
     """Checks if the frontend environment (ie node and yarn in bash) is available."""
+
     def ret(val, val0=None, val1=None):
         if versions:
             return val, val0, val1
@@ -140,7 +168,10 @@ def node_available(versions: bool = False, bypass_yarn: bool = False):
 
     def check(cmd):
         try:
-            return str(subprocess.check_output([cmd], stderr=subprocess.STDOUT, shell=True), encoding='utf-8').strip()
+            return str(
+                subprocess.check_output([cmd], stderr=subprocess.STDOUT, shell=True),
+                encoding='utf-8',
+            ).strip()
         except subprocess.CalledProcessError:
             return None
         except FileNotFoundError:
@@ -154,7 +185,9 @@ def node_available(versions: bool = False, bypass_yarn: bool = False):
 
     # Print a warning if node is available but yarn is not
     if node_version and not yarn_passes:
-        print('Node is available but yarn is not. Install yarn if you wish to build the frontend.')
+        print(
+            'Node is available but yarn is not. Install yarn if you wish to build the frontend.'
+        )
 
     # Return the result
     return ret(yarn_passes and node_version, node_version, yarn_version)
@@ -168,11 +201,13 @@ def check_file_existance(filename: str, overwrite: bool = False):
         overwrite (bool, optional): Overwrite the file without asking. Defaults to False.
     """
     if Path(filename).is_file() and overwrite is False:
-        response = input("Warning: file already exists. Do you want to overwrite? [y/N]: ")
+        response = input(
+            'Warning: file already exists. Do you want to overwrite? [y/N]: '
+        )
         response = str(response).strip().lower()
 
         if response not in ['y', 'yes']:
-            print("Cancelled export operation")
+            print('Cancelled export operation')
             sys.exit(1)
 
 
@@ -198,7 +233,9 @@ def install(c):
     # Install required Python packages with PIP
     c.run('pip3 install --upgrade pip')
     c.run('pip3 install --upgrade setuptools')
-    c.run('pip3 install --no-cache-dir --disable-pip-version-check -U -r requirements.txt')
+    c.run(
+        'pip3 install --no-cache-dir --disable-pip-version-check -U -r requirements.txt'
+    )
 
 
 @task(help={'tests': 'Set up test dataset at the end'})
@@ -210,12 +247,12 @@ def setup_dev(c, tests=False):
     c.run('pip3 install -U -r requirements-dev.txt')
 
     # Install pre-commit hook
-    print("Installing pre-commit for checks before git commits...")
+    print('Installing pre-commit for checks before git commits...')
     c.run('pre-commit install')
 
     # Update all the hooks
     c.run('pre-commit autoupdate')
-    print("pre-commit set up is done...")
+    print('pre-commit set up is done...')
 
     # Set up test-data if flag is set
     if tests:
@@ -232,19 +269,19 @@ def superuser(c):
 @task
 def rebuild_models(c):
     """Rebuild database models with MPTT structures."""
-    manage(c, "rebuild_models", pty=True)
+    manage(c, 'rebuild_models', pty=True)
 
 
 @task
 def rebuild_thumbnails(c):
     """Rebuild missing image thumbnails."""
-    manage(c, "rebuild_thumbnails", pty=True)
+    manage(c, 'rebuild_thumbnails', pty=True)
 
 
 @task
 def clean_settings(c):
     """Clean the setting tables of old settings."""
-    manage(c, "clean_settings")
+    manage(c, 'clean_settings')
 
 
 @task(help={'mail': "mail of the user who's MFA should be disabled"})
@@ -253,20 +290,16 @@ def remove_mfa(c, mail=''):
     if not mail:
         print('You must provide a users mail')
 
-    manage(c, f"remove_mfa {mail}")
+    manage(c, f'remove_mfa {mail}')
 
 
-@task(
-    help={
-        'frontend': 'Build the frontend',
-    }
-)
+@task(help={'frontend': 'Build the frontend'})
 def static(c, frontend=False):
     """Copies required static files to the STATIC_ROOT directory, as per Django requirements."""
-    manage(c, "prerender")
+    manage(c, 'prerender')
     if frontend and node_available():
         frontend_build(c)
-    manage(c, "collectstatic --no-input")
+    manage(c, 'collectstatic --no-input')
 
 
 @task
@@ -280,48 +313,49 @@ def translate_stats(c):
     try:
         manage(c, 'compilemessages', pty=True)
     except Exception:
-        print("WARNING: Translation files could not be compiled:")
+        print('WARNING: Translation files could not be compiled:')
 
     path = Path('InvenTree', 'script', 'translation_stats.py')
     c.run(f'python3 {path}')
 
 
 @task(post=[translate_stats])
-def translate(c):
+def translate(c, ignore_static=False, no_frontend=False):
     """Rebuild translation source files. Advanced use only!
 
     Note: This command should not be used on a local install,
     it is performed as part of the InvenTree translation toolchain.
     """
-    # Translate applicable .py / .html / .js / .tsx files
-    manage(c, "makemessages --all -e py,html,js --no-wrap")
-    manage(c, "compilemessages")
+    # Translate applicable .py / .html / .js files
+    manage(c, 'makemessages --all -e py,html,js --no-wrap')
+    manage(c, 'compilemessages')
 
-    if node_available():
+    if not no_frontend and node_available():
         frontend_install(c)
         frontend_trans(c)
         frontend_build(c)
 
     # Update static files
-    static(c)
+    if not ignore_static:
+        static(c)
 
 
 @task
 def backup(c):
     """Backup the database and media files."""
-    print("Backing up InvenTree database...")
-    manage(c, "dbbackup --noinput --clean --compress")
-    print("Backing up InvenTree media files...")
-    manage(c, "mediabackup --noinput --clean --compress")
+    print('Backing up InvenTree database...')
+    manage(c, 'dbbackup --noinput --clean --compress')
+    print('Backing up InvenTree media files...')
+    manage(c, 'mediabackup --noinput --clean --compress')
 
 
 @task
 def restore(c):
     """Restore the database and media files."""
-    print("Restoring InvenTree database...")
-    manage(c, "dbrestore --noinput --uncompress")
-    print("Restoring InvenTree media files...")
-    manage(c, "mediarestore --noinput --uncompress")
+    print('Restoring InvenTree database...')
+    manage(c, 'dbrestore --noinput --uncompress')
+    print('Restoring InvenTree media files...')
+    manage(c, 'mediarestore --noinput --uncompress')
 
 
 @task(post=[rebuild_models, rebuild_thumbnails])
@@ -330,16 +364,16 @@ def migrate(c):
 
     This is a critical step if the database schema have been altered!
     """
-    print("Running InvenTree database migrations...")
-    print("========================================")
+    print('Running InvenTree database migrations...')
+    print('========================================')
 
-    manage(c, "makemigrations")
-    manage(c, "migrate --noinput")
-    manage(c, "migrate --run-syncdb")
-    manage(c, "check")
+    manage(c, 'makemigrations')
+    manage(c, 'migrate --noinput')
+    manage(c, 'migrate --run-syncdb')
+    manage(c, 'check')
 
-    print("========================================")
-    print("InvenTree database migrations completed!")
+    print('========================================')
+    print('InvenTree database migrations completed!')
 
 
 @task(
@@ -347,9 +381,10 @@ def migrate(c):
     help={
         'skip_backup': 'Skip database backup step (advanced users)',
         'frontend': 'Force frontend compilation/download step (ignores INVENTREE_DOCKER)',
-    }
+        'no_frontend': 'Skip frontend compilation/download step',
+    },
 )
-def update(c, skip_backup=False, frontend: bool = False):
+def update(c, skip_backup=False, frontend: bool = False, no_frontend: bool = False):
     """Update InvenTree installation.
 
     This command should be invoked after source code has been updated,
@@ -378,8 +413,7 @@ def update(c, skip_backup=False, frontend: bool = False):
     # If:
     # - INVENTREE_DOCKER is set (by the docker image eg.) and not overridden by `--frontend` flag
     # - `--no-frontend` flag is set
-    # if (os.environ.get('INVENTREE_DOCKER', False) and not frontend) or no_frontend:
-    if not frontend:
+    if (os.environ.get('INVENTREE_DOCKER', False) and not frontend) or no_frontend:
         return
 
     # Decide if we should compile the frontend or try to download it
@@ -390,13 +424,27 @@ def update(c, skip_backup=False, frontend: bool = False):
 
 
 # Data tasks
-@task(help={
-    'filename': "Output filename (default = 'data.json')",
-    'overwrite': "Overwrite existing files without asking first (default = off/False)",
-    'include_permissions': "Include user and group permissions in the output file (filename) (default = off/False)",
-    'delete_temp': "Delete temporary files (containing permissions) at end of run. Note that this will delete temporary files from previous runs as well. (default = off/False)"
-})
-def export_records(c, filename='data.json', overwrite=False, include_permissions=False, delete_temp=False):
+@task(
+    help={
+        'filename': "Output filename (default = 'data.json')",
+        'overwrite': 'Overwrite existing files without asking first (default = False)',
+        'include_permissions': 'Include user and group permissions in the output file (default = False)',
+        'include_tokens': 'Include API tokens in the output file (default = False)',
+        'exclude_plugins': 'Exclude plugin data from the output file (default = False)',
+        'include_sso': 'Include SSO token data in the output file (default = False)',
+        'retain_temp': 'Retain temporary files (containing permissions) at end of process (default = False)',
+    }
+)
+def export_records(
+    c,
+    filename='data.json',
+    overwrite=False,
+    include_permissions=False,
+    include_tokens=False,
+    exclude_plugins=False,
+    include_sso=False,
+    retain_temp=False,
+):
     """Export all database records to a file.
 
     Write data to the file defined by filename.
@@ -422,44 +470,58 @@ def export_records(c, filename='data.json', overwrite=False, include_permissions
 
     check_file_existance(filename, overwrite)
 
-    tmpfile = f"{filename}.tmp"
+    tmpfile = f'{filename}.tmp'
 
-    cmd = f"dumpdata --indent 2 --output '{tmpfile}' {content_excludes()}"
+    excludes = content_excludes(
+        allow_tokens=include_tokens,
+        allow_plugins=not exclude_plugins,
+        allow_sso=include_sso,
+    )
+
+    cmd = f"dumpdata --natural-foreign --indent 2 --output '{tmpfile}' {excludes}"
 
     # Dump data to temporary file
     manage(c, cmd, pty=True)
 
-    print("Running data post-processing step...")
+    print('Running data post-processing step...')
 
     # Post-process the file, to remove any "permissions" specified for a user or group
-    with open(tmpfile, "r") as f_in:
+    with open(tmpfile, 'r') as f_in:
         data = json.loads(f_in.read())
 
     if include_permissions is False:
         for entry in data:
-            if "model" in entry:
-
+            if 'model' in entry:
                 # Clear out any permissions specified for a group
-                if entry["model"] == "auth.group":
-                    entry["fields"]["permissions"] = []
+                if entry['model'] == 'auth.group':
+                    entry['fields']['permissions'] = []
 
                 # Clear out any permissions specified for a user
-                if entry["model"] == "auth.user":
-                    entry["fields"]["user_permissions"] = []
+                if entry['model'] == 'auth.user':
+                    entry['fields']['user_permissions'] = []
 
     # Write the processed data to file
-    with open(filename, "w") as f_out:
+    with open(filename, 'w') as f_out:
         f_out.write(json.dumps(data, indent=2))
 
-    print("Data export completed")
+    print('Data export completed')
 
-    if delete_temp is True:
-        print("Removing temporary file")
+    if not retain_temp:
+        print('Removing temporary files')
         os.remove(tmpfile)
 
 
-@task(help={'filename': 'Input filename', 'clear': 'Clear existing data before import'}, post=[rebuild_models, rebuild_thumbnails])
-def import_records(c, filename='data.json', clear=False):
+@task(
+    help={
+        'filename': 'Input filename',
+        'clear': 'Clear existing data before import',
+        'retain_temp': 'Retain temporary files at end of process (default = False)',
+    },
+    post=[rebuild_models, rebuild_thumbnails],
+)
+def import_records(
+    c, filename='data.json', clear: bool = False, retain_temp: bool = False
+):
     """Import database records from a file."""
     # Get an absolute path to the supplied filename
     if not os.path.isabs(filename):
@@ -474,32 +536,69 @@ def import_records(c, filename='data.json', clear=False):
 
     print(f"Importing database records from '{filename}'")
 
-    # Pre-process the data, to remove any "permissions" specified for a user or group
-    tmpfile = f"{filename}.tmp.json"
+    # We need to load 'auth' data (users / groups) *first*
+    # This is due to the users.owner model, which has a ContentType foreign key
+    authfile = f'{filename}.auth.json'
 
-    with open(filename, "r") as f_in:
-        data = json.loads(f_in.read())
+    # Pre-process the data, to remove any "permissions" specified for a user or group
+    datafile = f'{filename}.data.json'
+
+    with open(filename, 'r') as f_in:
+        try:
+            data = json.loads(f_in.read())
+        except json.JSONDecodeError as exc:
+            print(f'Error: Failed to decode JSON file: {exc}')
+            sys.exit(1)
+
+    auth_data = []
+    load_data = []
 
     for entry in data:
-        if "model" in entry:
-
+        if 'model' in entry:
             # Clear out any permissions specified for a group
-            if entry["model"] == "auth.group":
-                entry["fields"]["permissions"] = []
+            if entry['model'] == 'auth.group':
+                entry['fields']['permissions'] = []
 
             # Clear out any permissions specified for a user
-            if entry["model"] == "auth.user":
-                entry["fields"]["user_permissions"] = []
+            if entry['model'] == 'auth.user':
+                entry['fields']['user_permissions'] = []
+
+            # Save auth data for later
+            if entry['model'].startswith('auth.'):
+                auth_data.append(entry)
+            else:
+                load_data.append(entry)
+        else:
+            print('Warning: Invalid entry in data file')
+            print(entry)
+
+    # Write the auth file data
+    with open(authfile, 'w') as f_out:
+        f_out.write(json.dumps(auth_data, indent=2))
 
     # Write the processed data to the tmp file
-    with open(tmpfile, "w") as f_out:
-        f_out.write(json.dumps(data, indent=2))
+    with open(datafile, 'w') as f_out:
+        f_out.write(json.dumps(load_data, indent=2))
 
-    cmd = f"loaddata '{tmpfile}' -i {content_excludes()}"
+    excludes = content_excludes(allow_auth=False)
+
+    # Import auth models first
+    print('Importing user auth data...')
+    cmd = f"loaddata '{authfile}'"
+    manage(c, cmd, pty=True)
+
+    # Import everything else next
+    print('Importing database records...')
+    cmd = f"loaddata '{datafile}' -i {excludes}"
 
     manage(c, cmd, pty=True)
 
-    print("Data import completed")
+    if not retain_temp:
+        print('Removing temporary files')
+        os.remove(datafile)
+        os.remove(authfile)
+
+    print('Data import completed')
 
 
 @task
@@ -508,7 +607,7 @@ def delete_data(c, force=False):
 
     Warning: This will REALLY delete all records in the database!!
     """
-    print("Deleting all data from InvenTree database...")
+    print('Deleting all data from InvenTree database...')
 
     if force:
         manage(c, 'flush --noinput')
@@ -530,32 +629,26 @@ def import_fixtures(c):
     fixtures = [
         # Build model
         'build',
-
         # Common models
         'settings',
-
         # Company model
         'company',
         'price_breaks',
         'supplier_part',
-
         # Order model
         'order',
-
         # Part model
         'bom',
         'category',
         'params',
         'part',
         'test_templates',
-
         # Stock model
         'location',
         'stock_tests',
         'stock',
-
         # Users
-        'users'
+        'users',
     ]
 
     command = 'loaddata ' + ' '.join(fixtures)
@@ -567,16 +660,16 @@ def import_fixtures(c):
 @task
 def wait(c):
     """Wait until the database connection is ready."""
-    return manage(c, "wait_for_db")
+    return manage(c, 'wait_for_db')
 
 
 @task(pre=[wait], help={'address': 'Server address:port (default=127.0.0.1:8000)'})
-def server(c, address="127.0.0.1:8000"):
+def server(c, address='127.0.0.1:8000'):
     """Launch a (development) server using Django's in-built webserver.
 
     Note: This is *not* sufficient for a production installation.
     """
-    manage(c, "runserver {address}".format(address=address), pty=True)
+    manage(c, 'runserver {address}'.format(address=address), pty=True)
 
 
 @task(pre=[wait])
@@ -589,7 +682,7 @@ def worker(c):
 @task
 def render_js_files(c):
     """Render templated javascript files (used for static testing)."""
-    manage(c, "test InvenTree.ci_render_js")
+    manage(c, 'test InvenTree.ci_render_js')
 
 
 @task(post=[translate_stats, static, server])
@@ -607,40 +700,44 @@ def test_translations(c):
     django.setup()
 
     # Add language
-    print("Add dummy language...")
-    print("========================================")
-    manage(c, "makemessages -e py,html,js --no-wrap -l xx")
+    print('Add dummy language...')
+    print('========================================')
+    manage(c, 'makemessages -e py,html,js --no-wrap -l xx')
 
     # change translation
-    print("Fill in dummy translations...")
-    print("========================================")
+    print('Fill in dummy translations...')
+    print('========================================')
 
     file_path = pathlib.Path(settings.LOCALE_PATHS[0], 'xx', 'LC_MESSAGES', 'django.po')
     new_file_path = str(file_path) + '_new'
 
     # compile regex
     reg = re.compile(
-        r"[a-zA-Z0-9]{1}" +  # match any single letter and number  # noqa: W504
-        r"(?![^{\(\<]*[}\)\>])" +  # that is not inside curly brackets, brackets or a tag  # noqa: W504
-        r"(?<![^\%][^\(][)][a-z])" +  # that is not a specially formatted variable with singles  # noqa: W504
-        r"(?![^\\][\n])"  # that is not a newline
+        r'[a-zA-Z0-9]{1}' +  # match any single letter and number  # noqa: W504
+        r'(?![^{\(\<]*[}\)\>])' +  # that is not inside curly brackets, brackets or a tag  # noqa: W504
+        r'(?<![^\%][^\(][)][a-z])' +  # that is not a specially formatted variable with singles  # noqa: W504
+        r'(?![^\\][\n])'  # that is not a newline
     )
     last_string = ''
 
     # loop through input file lines
-    with open(file_path, "rt") as file_org:
-        with open(new_file_path, "wt") as file_new:
+    with open(file_path, 'rt') as file_org:
+        with open(new_file_path, 'wt') as file_new:
             for line in file_org:
                 if line.startswith('msgstr "'):
                     # write output -> replace regex matches with x in the read in (multi)string
                     file_new.write(f'msgstr "{reg.sub("x", last_string[7:-2])}"\n')
-                    last_string = ""  # reset (multi)string
+                    last_string = ''  # reset (multi)string
                 elif line.startswith('msgid "'):
-                    last_string = last_string + line  # a new translatable string starts -> start append
+                    last_string = (
+                        last_string + line
+                    )  # a new translatable string starts -> start append
                     file_new.write(line)
                 else:
                     if last_string:
-                        last_string = last_string + line  # a string is being read in -> continue appending
+                        last_string = (
+                            last_string + line
+                        )  # a string is being read in -> continue appending
                     file_new.write(line)
 
     # change out translation files
@@ -648,9 +745,9 @@ def test_translations(c):
     new_file_path.rename(file_path)
 
     # compile languages
-    print("Compile languages ...")
-    print("========================================")
-    manage(c, "compilemessages")
+    print('Compile languages ...')
+    print('========================================')
+    manage(c, 'compilemessages')
 
     # reset cwd
     os.chdir(base_path)
@@ -668,7 +765,9 @@ def test_translations(c):
         'coverage': 'Run code coverage analysis (requires coverage package)',
     }
 )
-def test(c, disable_pty=False, runtest='', migrations=False, report=False, coverage=False):
+def test(
+    c, disable_pty=False, runtest='', migrations=False, report=False, coverage=False
+):
     """Run unit-tests for InvenTree codebase.
 
     To run only certain test, use the argument --runtest.
@@ -713,7 +812,7 @@ def test(c, disable_pty=False, runtest='', migrations=False, report=False, cover
 
 
 @task(help={'dev': 'Set up development environment at the end'})
-def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset"):
+def setup_test(c, ignore_update=False, dev=False, path='inventree-demo-dataset'):
     """Setup a testing environment."""
     from InvenTree.InvenTree.config import get_media_dir
 
@@ -722,41 +821,43 @@ def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset")
 
     # Remove old data directory
     if os.path.exists(path):
-        print("Removing old data ...")
+        print('Removing old data ...')
         c.run(f'rm {path} -r')
 
     # Get test data
-    print("Cloning demo dataset ...")
+    print('Cloning demo dataset ...')
     c.run(f'git clone https://github.com/inventree/demo-dataset {path} -v --depth=1')
-    print("========================================")
+    print('========================================')
 
     # Make sure migrations are done - might have just deleted sqlite database
     if not ignore_update:
         migrate(c)
 
     # Load data
-    print("Loading database records ...")
+    print('Loading database records ...')
     import_records(c, filename=f'{path}/inventree_data.json', clear=True)
 
     # Copy media files
-    print("Copying media files ...")
+    print('Copying media files ...')
     src = Path(path).joinpath('media').resolve()
     dst = get_media_dir()
 
     shutil.copytree(src, dst, dirs_exist_ok=True)
 
-    print("Done setting up test environment...")
-    print("========================================")
+    print('Done setting up test environment...')
+    print('========================================')
 
     # Set up development setup if flag is set
     if dev:
         setup_dev(c)
 
 
-@task(help={
-    'filename': "Output filename (default = 'schema.yml')",
-    'overwrite': "Overwrite existing files without asking first (default = off/False)",
-})
+@task(
+    help={
+        'filename': "Output filename (default = 'schema.yml')",
+        'overwrite': 'Overwrite existing files without asking first (default = off/False)',
+    }
+)
 def schema(c, filename='schema.yml', overwrite=False):
     """Export current API schema."""
     check_file_existance(filename, overwrite)
@@ -773,7 +874,8 @@ def version(c):
     # Gather frontend version information
     _, node, yarn = node_available(versions=True)
 
-    print(f"""
+    print(
+        f"""
 InvenTree - inventree.org
 The Open-Source Inventory Management System\n
 
@@ -792,13 +894,16 @@ Node        {node if node else 'N/A'}
 Yarn        {yarn if yarn else 'N/A'}
 
 Commit hash:{InvenTreeVersion.inventreeCommitHash()}
-Commit date:{InvenTreeVersion.inventreeCommitDate()}""")
+Commit date:{InvenTreeVersion.inventreeCommitDate()}"""
+    )
     if len(sys.argv) == 1 and sys.argv[0].startswith('/opt/inventree/env/lib/python'):
-        print("""
+        print(
+            """
 You are probably running the package installer / single-line installer. Please mentioned that in any bug reports!
 
 Use '--list' for a list of available commands
-Use '--help' for help on a specific command""")
+Use '--help' for help on a specific command"""
+        )
 
 
 @task()
@@ -826,8 +931,8 @@ def frontend_install(c):
     Args:
         c: Context variable
     """
-    print("Installing frontend dependencies")
-    yarn(c, "yarn install")
+    print('Installing frontend dependencies')
+    yarn(c, 'yarn install')
 
 
 @task
@@ -837,9 +942,9 @@ def frontend_trans(c):
     Args:
         c: Context variable
     """
-    print("Compiling frontend translations")
-    yarn(c, "yarn run extract")
-    yarn(c, "yarn run compile")
+    print('Compiling frontend translations')
+    yarn(c, 'yarn run extract')
+    yarn(c, 'yarn run compile')
 
 
 @task
@@ -849,8 +954,8 @@ def frontend_build(c):
     Args:
         c: Context variable
     """
-    print("Building frontend")
-    yarn(c, "yarn run build --emptyOutDir")
+    print('Building frontend')
+    yarn(c, 'yarn run build --emptyOutDir')
 
 
 @task
@@ -860,19 +965,29 @@ def frontend_dev(c):
     Args:
         c: Context variable
     """
-    print("Starting frontend development server")
-    yarn(c, "yarn run dev")
+    print('Starting frontend development server')
+    yarn(c, 'yarn run dev')
 
 
-@task(help={
-    'ref': "git ref, default: current git ref",
-    'tag': "git tag to look for release",
-    'file': "destination to frontend-build.zip file",
-    'repo': "GitHub repository, default: InvenTree/inventree",
-    'extract': "Also extract and place at the correct destination, default: True",
-    'clean': "Delete old files from InvenTree/web/static/web first, default: True",
-})
-def frontend_download(c, ref=None, tag=None, file=None, repo="InvenTree/inventree", extract=True, clean=True):
+@task(
+    help={
+        'ref': 'git ref, default: current git ref',
+        'tag': 'git tag to look for release',
+        'file': 'destination to frontend-build.zip file',
+        'repo': 'GitHub repository, default: InvenTree/inventree',
+        'extract': 'Also extract and place at the correct destination, default: True',
+        'clean': 'Delete old files from InvenTree/web/static/web first, default: True',
+    }
+)
+def frontend_download(
+    c,
+    ref=None,
+    tag=None,
+    file=None,
+    repo='InvenTree/inventree',
+    extract=True,
+    clean=True,
+):
     """Download a pre-build frontend from GitHub if you dont want to install nodejs on your machine.
 
     There are 3 possibilities to install the frontend:
@@ -894,7 +1009,7 @@ def frontend_download(c, ref=None, tag=None, file=None, repo="InvenTree/inventre
     import requests
 
     # globals
-    default_headers = {"Accept": "application/vnd.github.v3+json"}
+    default_headers = {'Accept': 'application/vnd.github.v3+json'}
 
     # helper functions
     def find_resource(resource, key, value):
@@ -908,30 +1023,34 @@ def frontend_download(c, ref=None, tag=None, file=None, repo="InvenTree/inventre
         if not extract:
             return
 
-        dest_path = Path(__file__).parent / "InvenTree/web/static/web"
+        dest_path = Path(__file__).parent / 'InvenTree/web/static/web'
 
         # if clean, delete static/web directory
         if clean:
             shutil.rmtree(dest_path, ignore_errors=True)
             os.makedirs(dest_path)
-            print(f"Cleaned directory: {dest_path}")
+            print(f'Cleaned directory: {dest_path}')
 
         # unzip build to static folder
-        with ZipFile(file, "r") as zip_ref:
+        with ZipFile(file, 'r') as zip_ref:
             zip_ref.extractall(dest_path)
 
-        print(f"Unzipped downloaded frontend build to: {dest_path}")
+        print(f'Unzipped downloaded frontend build to: {dest_path}')
 
     def handle_download(url):
         # download frontend-build.zip to temporary file
-        with requests.get(url, headers=default_headers, stream=True, allow_redirects=True) as response, NamedTemporaryFile(suffix=".zip") as dst:
+        with requests.get(
+            url, headers=default_headers, stream=True, allow_redirects=True
+        ) as response, NamedTemporaryFile(suffix='.zip') as dst:
             response.raise_for_status()
 
             # auto decode the gzipped raw data
-            response.raw.read = functools.partial(response.raw.read, decode_content=True)
-            with open(dst.name, "wb") as f:
+            response.raw.read = functools.partial(
+                response.raw.read, decode_content=True
+            )
+            with open(dst.name, 'wb') as f:
                 shutil.copyfileobj(response.raw, f)
-            print(f"Downloaded frontend build to temporary file: {dst.name}")
+            print(f'Downloaded frontend build to temporary file: {dst.name}')
 
             handle_extract(dst.name)
 
@@ -942,51 +1061,72 @@ def frontend_download(c, ref=None, tag=None, file=None, repo="InvenTree/inventre
 
     # check arguments
     if ref is not None and tag is not None:
-        print("[ERROR] Do not set ref and tag.")
+        print('[ERROR] Do not set ref and tag.')
         return
 
     if ref is None and tag is None:
         try:
-            ref = subprocess.check_output(["git", "rev-parse", "HEAD"], encoding="utf-8").strip()
+            ref = subprocess.check_output(
+                ['git', 'rev-parse', 'HEAD'], encoding='utf-8'
+            ).strip()
         except Exception:
             print("[ERROR] Cannot get current ref via 'git rev-parse HEAD'")
             return
 
     if ref is None and tag is None:
-        print("[ERROR] Either ref or tag needs to be set.")
+        print('[ERROR] Either ref or tag needs to be set.')
 
     if tag:
-        tag = tag.lstrip("v")
+        tag = tag.lstrip('v')
         try:
-            handle_download(f"https://github.com/{repo}/releases/download/{tag}/frontend-build.zip")
+            handle_download(
+                f'https://github.com/{repo}/releases/download/{tag}/frontend-build.zip'
+            )
         except Exception as e:
             if not isinstance(e, requests.HTTPError):
                 raise e
-            print(f"""[ERROR] An Error occurred. Unable to download frontend build, release or build does not exist,
+            print(
+                f"""[ERROR] An Error occurred. Unable to download frontend build, release or build does not exist,
 try downloading the frontend-build.zip yourself via: https://github.com/{repo}/releases
-Then try continuing by running: invoke frontend-download --file <path-to-downloaded-zip-file>""")
+Then try continuing by running: invoke frontend-download --file <path-to-downloaded-zip-file>"""
+            )
 
         return
 
     if ref:
         # get workflow run from all workflow runs on that particular ref
-        workflow_runs = requests.get(f"https://api.github.com/repos/{repo}/actions/runs?head_sha={ref}", headers=default_headers).json()
+        workflow_runs = requests.get(
+            f'https://api.github.com/repos/{repo}/actions/runs?head_sha={ref}',
+            headers=default_headers,
+        ).json()
 
-        if not (qc_run := find_resource(workflow_runs["workflow_runs"], "name", "QC")):
-            print("[ERROR] Cannot find any workflow runs for current sha")
+        if not (qc_run := find_resource(workflow_runs['workflow_runs'], 'name', 'QC')):
+            print('[ERROR] Cannot find any workflow runs for current sha')
             return
-        print(f"Found workflow {qc_run['name']} (run {qc_run['run_number']}-{qc_run['run_attempt']})")
+        print(
+            f"Found workflow {qc_run['name']} (run {qc_run['run_number']}-{qc_run['run_attempt']})"
+        )
 
         # get frontend-build artifact from all artifacts available for this workflow run
-        artifacts = requests.get(qc_run["artifacts_url"], headers=default_headers).json()
-        if not (frontend_artifact := find_resource(artifacts["artifacts"], "name", "frontend-build")):
-            print("[ERROR] Cannot find frontend-build.zip attachment for current sha")
+        artifacts = requests.get(
+            qc_run['artifacts_url'], headers=default_headers
+        ).json()
+        if not (
+            frontend_artifact := find_resource(
+                artifacts['artifacts'], 'name', 'frontend-build'
+            )
+        ):
+            print('[ERROR] Cannot find frontend-build.zip attachment for current sha')
             return
-        print(f"Found artifact {frontend_artifact['name']} with id {frontend_artifact['id']} ({frontend_artifact['size_in_bytes']/1e6:.2f}MB).")
+        print(
+            f"Found artifact {frontend_artifact['name']} with id {frontend_artifact['id']} ({frontend_artifact['size_in_bytes']/1e6:.2f}MB)."
+        )
 
-        print(f"""
+        print(
+            f"""
 GitHub doesn't allow artifact downloads from anonymous users. Either download the following file
 via your signed in browser, or consider using a point release download via invoke frontend-download --tag <git-tag>
 
     Download: https://github.com/{repo}/suites/{qc_run['check_suite_id']}/artifacts/{frontend_artifact['id']} manually and
-    continue by running: invoke frontend-download --file <path-to-downloaded-zip-file>""")
+    continue by running: invoke frontend-download --file <path-to-downloaded-zip-file>"""
+        )