1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-03-16 05:23:33 +00:00

Improve data source and dispatch endpoints performances

This commit is contained in:
Jérémie Pardou 2025-02-19 09:42:10 +00:00
parent 752963f33c
commit a97987bfc0
50 changed files with 813 additions and 332 deletions
backend
changelog/entries/unreleased/refactor
docker-compose.yml
enterprise/backend/src/baserow_enterprise
premium/backend/src/baserow_premium
web-frontend
modules/builder/store
test/unit/builder/components/elements/components/__snapshots__

View file

@ -110,6 +110,7 @@ MIDDLEWARE = [
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"baserow.core.cache.LocalCacheMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"baserow.api.user_sources.middleware.AddUserSourceUserMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
@ -1344,3 +1345,5 @@ BASEROW_DEFAULT_ZIP_COMPRESS_LEVEL = 5
BASEROW_MAX_HEALTHY_CELERY_QUEUE_SIZE = int(
os.getenv("BASEROW_MAX_HEALTHY_CELERY_QUEUE_SIZE", "") or 10
)
BASEROW_USE_LOCAL_CACHE = str_to_bool(os.getenv("BASEROW_USE_LOCAL_CACHE", "true"))

View file

@ -186,7 +186,11 @@ class DataSourcesView(APIView):
page = PageHandler().get_page(page_id)
before = DataSourceHandler().get_data_source(before_id) if before_id else None
before = (
DataSourceHandler().get_data_source(before_id, specific=False)
if before_id
else None
)
service_type = service_type_registry.get(type_name) if type_name else None
@ -423,7 +427,7 @@ class MoveDataSourceView(APIView):
before = None
if before_id:
before = DataSourceHandler().get_data_source(before_id)
before = DataSourceHandler().get_data_source(before_id, specific=False)
moved_data_source = DataSourceService().move_data_source(
request.user, data_source, before

View file

@ -201,7 +201,9 @@ class DataSourceDataProviderType(DataProviderType):
return {}
try:
data_source = DataSourceHandler().get_data_source(data_source_id)
data_source = DataSourceHandler().get_data_source(
data_source_id, with_cache=True
)
except DataSourceDoesNotExist as exc:
# The data source has probably been deleted
raise InvalidBaserowFormula() from exc
@ -272,7 +274,9 @@ class DataSourceContextDataProviderType(DataProviderType):
return {}
try:
data_source = DataSourceHandler().get_data_source(data_source_id)
data_source = DataSourceHandler().get_data_source(
data_source_id, with_cache=True
)
except DataSourceDoesNotExist as exc:
# The data source has probably been deleted
raise InvalidBaserowFormula() from exc
@ -376,7 +380,9 @@ class CurrentRecordDataProviderType(DataProviderType):
return {}
try:
data_source = DataSourceHandler().get_data_source(data_source_id)
data_source = DataSourceHandler().get_data_source(
data_source_id, with_cache=True
)
except DataSourceDoesNotExist as exc:
# The data source is probably not accessible so we raise an invalid formula
raise InvalidBaserowFormula() from exc

View file

@ -24,6 +24,8 @@ CACHE_KEY_PREFIX = "used_properties_for_page"
User = get_user_model()
SENTINEL = "__no_results__"
class BuilderDispatchContext(DispatchContext):
own_properties = [

View file

@ -17,6 +17,7 @@ from baserow.contrib.builder.data_sources.models import DataSource
from baserow.contrib.builder.formula_importer import import_formula
from baserow.contrib.builder.pages.models import Page
from baserow.contrib.builder.types import DataSourceDict
from baserow.core.cache import local_cache
from baserow.core.integrations.models import Integration
from baserow.core.integrations.registries import integration_type_registry
from baserow.core.services.handler import ServiceHandler
@ -36,13 +37,50 @@ class DataSourceHandler:
self.service_handler = ServiceHandler()
def get_data_source(
self, data_source_id: int, base_queryset: Optional[QuerySet] = None, cache=None
self,
data_source_id: int,
base_queryset: Optional[QuerySet] = None,
specific=True,
with_cache=False,
) -> DataSource:
"""
Returns a data_source instance from the database.
:param data_source_id: The ID of the data_source.
:param base_queryset: The base queryset to use to build the query.
:param specific: Return the specific version of related objects like the
service and the integration
:raises DataSourceDoesNotExist: If the data_source can't be found.
:param with_cache: Whether this method should use the short
cache for data_sources.
:return: The data_source instance.
"""
if with_cache and not base_queryset:
return local_cache.get(
f"ab_data_source_{data_source_id}_{specific}",
lambda: self._get_data_source(
data_source_id, base_queryset, specific=specific
),
)
else:
return self._get_data_source(
data_source_id, base_queryset, specific=specific
)
def _get_data_source(
self,
data_source_id: int,
base_queryset: Optional[QuerySet] = None,
specific=True,
) -> DataSource:
"""
Base version of the get_data_source without the caching capabilities.
:param data_source_id: The ID of the data_source.
:param base_queryset: The base queryset to use to build the query.
:param specific: Return the specific version of related objects like the
service and the integration
:raises DataSourceDoesNotExist: If the data_source can't be found.
:return: The data_source instance.
"""
@ -51,12 +89,24 @@ class DataSourceHandler:
base_queryset if base_queryset is not None else DataSource.objects.all()
)
queryset = queryset.select_related("page__builder__workspace")
try:
data_source = queryset.select_related(
"page", "page__builder", "page__builder__workspace", "service"
).get(id=data_source_id)
except DataSource.DoesNotExist:
raise DataSourceDoesNotExist()
if specific:
data_source = queryset.get(id=data_source_id)
if data_source.service_id:
specific_service = ServiceHandler().get_service(
data_source.service_id, specific=True
)
data_source.__class__.service.field.set_cached_value(
data_source, specific_service
)
else:
data_source = queryset.select_related("service__integration").get(
id=data_source_id
)
except DataSource.DoesNotExist as exc:
raise DataSourceDoesNotExist() from exc
return data_source
@ -83,26 +133,23 @@ class DataSourceHandler:
base_queryset=queryset,
)
def _query_data_sources(self, base_queryset: QuerySet, specific=True):
def _query_data_sources(
self, base_queryset: QuerySet, specific=True, with_cache=False
):
"""
Query data sources from the base queryset.
:param base_queryset: The base QuerySet to query from.
:param specific: A boolean flag indicating whether to include specific service
instance.
:param with_cache: Whether this method should populate the short
cache for data_sources.
:return: A list of queried data sources.
"""
data_source_queryset = base_queryset.select_related(
"service",
"page__builder__workspace",
"service__integration__application",
)
data_source_queryset = base_queryset.select_related("page__builder__workspace")
if specific:
data_source_queryset = data_source_queryset.select_related(
"service__content_type"
)
data_sources = list(data_source_queryset.all())
# Get all service ids to get them from DB in one query
@ -124,9 +171,19 @@ class DataSourceHandler:
if data_source.service_id:
data_source.service = specific_services_map[data_source.service_id]
return data_sources
else:
return data_source_queryset.all()
data_source_queryset.select_related(
"service__integration__application",
)
data_sources = data_source_queryset.all()
if with_cache:
for ds in data_sources:
local_cache.get(
f"ab_data_source_{ds.id}_{specific}",
ds,
)
return data_sources
def get_data_sources(
self,
@ -134,6 +191,7 @@ class DataSourceHandler:
base_queryset: Optional[QuerySet] = None,
with_shared: Optional[bool] = False,
specific: Optional[bool] = True,
with_cache=False,
) -> Union[QuerySet[DataSource], Iterable[DataSource]]:
"""
Gets all the specific data_sources of a given page.
@ -144,6 +202,8 @@ class DataSourceHandler:
on the same builder.
:param specific: If True, return the specific version of the service related
to the data source
:param with_cache: Whether this method should populate the short
cache for data_sources.
:return: The data_sources of that page.
"""
@ -159,13 +219,18 @@ class DataSourceHandler:
else:
data_source_queryset = data_source_queryset.filter(page=page)
return self._query_data_sources(data_source_queryset, specific=specific)
return self._query_data_sources(
data_source_queryset,
specific=specific,
with_cache=with_cache,
)
def get_builder_data_sources(
self,
builder: "Builder",
base_queryset: Optional[QuerySet] = None,
specific: Optional[bool] = True,
with_cache=False,
) -> Union[QuerySet[DataSource], Iterable[DataSource]]:
"""
Gets all the specific data_sources of a given builder.
@ -174,6 +239,8 @@ class DataSourceHandler:
:param base_queryset: The base queryset to use to build the query.
:param specific: If True, return the specific version of the service related
to the data source
:param with_cache: Whether this method should populate the short
cache for data_sources.
:return: The data_sources of that builder.
"""
@ -183,7 +250,11 @@ class DataSourceHandler:
data_source_queryset = data_source_queryset.filter(page__builder=builder)
return self._query_data_sources(data_source_queryset, specific=specific)
return self._query_data_sources(
data_source_queryset,
specific=specific,
with_cache=with_cache,
)
def get_data_sources_with_cache(
self,
@ -192,26 +263,25 @@ class DataSourceHandler:
specific: bool = True,
):
"""
Gets all the specific data_sources of a given page. This version cache the
Gets all the data sources of a given page. This version cache the
data sources of the page onto the page object to improve perfs.
:param page: The page that holds the data_source.
:param base_queryset: The base queryset to use to build the query.
:param specific: If True, return the specific version of the service related
to the integration
to the data source
:return: The data_sources of the page.
"""
if not hasattr(page, "_data_sources"):
data_sources = DataSourceHandler().get_data_sources(
return local_cache.get(
f"ab_data_sources_{page.id}_{specific}",
lambda: DataSourceHandler().get_data_sources(
page,
base_queryset=base_queryset,
specific=specific,
with_shared=True,
)
setattr(page, "_data_sources", data_sources)
return getattr(page, "_data_sources")
),
)
def get_data_source_with_cache(
self,

View file

@ -89,13 +89,15 @@ class DataSourceService:
return self.handler.get_data_sources(page, base_queryset=user_data_sources)
def get_builder_data_sources(
self, user: AbstractUser, builder: "Builder"
self, user: AbstractUser, builder: "Builder", with_cache=False
) -> List[DataSource]:
"""
Gets all the data_sources of a given builder visible to the given user.
:param user: The user trying to get the data_sources.
:param page: The builder that holds the data_sources.
:param with_cache: Whether this method should populate the short
cache for data_sources.
:return: The data_sources of that builder.
"""
@ -107,7 +109,9 @@ class DataSourceService:
)
return self.handler.get_builder_data_sources(
builder, base_queryset=user_data_sources
builder,
base_queryset=user_data_sources,
with_cache=with_cache,
)
def create_data_source(

View file

@ -57,7 +57,7 @@ class DomainHandler:
"""
if base_queryset is None:
base_queryset = Domain.objects
base_queryset = Domain.objects.all()
return specific_iterator(base_queryset.filter(builder=builder))
@ -73,7 +73,7 @@ class DomainHandler:
try:
domain = (
Domain.objects.exclude(published_to=None)
.select_related("published_to", "builder")
.select_related("published_to", "builder__workspace")
.only("published_to", "builder")
.get(domain_name=domain_name)
)

View file

@ -1,3 +1,5 @@
import functools
from django.contrib.auth import get_user_model
from baserow.contrib.builder.data_sources.operations import (
@ -11,6 +13,7 @@ from baserow.contrib.builder.workflow_actions.operations import (
DispatchBuilderWorkflowActionOperationType,
ListBuilderWorkflowActionsPageOperationType,
)
from baserow.core.cache import local_cache
from baserow.core.operations import ReadApplicationOperationType
from baserow.core.registries import PermissionManagerType, operation_type_registry
from baserow.core.subjects import AnonymousUserSubjectType, UserSubjectType
@ -56,25 +59,54 @@ class AllowPublicBuilderManagerType(PermissionManagerType):
ListUserSourcesApplicationOperationType.type,
]
def get_builder_from_id(self, builder_id):
"""
Returns the builder for the given id. Can be a cached version.
"""
def get_builder_if_exists():
try:
return Builder.objects.get(id=builder_id)
except Builder.DoesNotExist:
return None
return local_cache.get(f"ab_builder_{builder_id}", get_builder_if_exists)
def get_builder_from_instance(self, instance, property_name):
"""
Returns the builder from the instance at the given property. The value can be
cached.
"""
prop_id_name = f"{property_name}_id"
if getattr(instance.__class__, property_name).is_cached(instance):
return local_cache.get(
f"ab_builder_{getattr(instance, prop_id_name)}",
lambda: getattr(instance, property_name),
)
else:
return self.get_builder_from_id(getattr(instance, prop_id_name))
def check_multiple_permissions(self, checks, workspace=None, include_trash=False):
result = {}
for check in checks:
operation_type = operation_type_registry.get(check.operation_name)
if operation_type.type in self.page_level_operations:
builder = check.context.builder
builder = self.get_builder_from_instance(check.context, "builder")
elif operation_type.type in self.sub_page_level_operations:
builder = check.context.page.builder
builder = self.get_builder_from_instance(check.context.page, "builder")
elif (
operation_type.type in self.sub_application_level_operations
and isinstance(check.context.application.specific, Builder)
and self.get_builder_from_id(check.context.application_id)
):
builder = check.context.application.specific
builder = self.get_builder_from_id(check.context.application_id)
elif (
operation_type.type in self.application_level_operations
and isinstance(check.context.specific, Builder)
and self.get_builder_from_id(check.context.id)
):
builder = check.context.specific
builder = self.get_builder_from_id(check.context.id)
else:
continue
@ -100,7 +132,18 @@ class AllowPublicBuilderManagerType(PermissionManagerType):
# give access to specific data.
continue
if DomainHandler().get_domain_for_builder(builder) is not None:
def is_public_callback(b):
return (
b.workspace is None
and DomainHandler().get_domain_for_builder(b) is not None
)
is_public = local_cache.get(
f"ab_is_public_builder_{builder.id}",
functools.partial(is_public_callback, builder),
)
if is_public:
# it's a public builder, we allow it.
result[check] = True

View file

@ -124,9 +124,7 @@ class ElementHandler:
try:
element = (
queryset.select_related(
"page", "page__builder", "page__builder__workspace"
)
queryset.select_related("page__builder__workspace")
.get(id=element_id)
.specific
)
@ -239,8 +237,7 @@ class ElementHandler:
"""
if specific:
queryset = base_queryset.select_related("content_type")
elements = specific_iterator(queryset)
elements = specific_iterator(base_queryset)
else:
elements = base_queryset

View file

@ -502,7 +502,7 @@ class CollectionElementTypeMixin:
# current instance
data_source_id = instance.data_source_id or kwargs.get("data_source_id", None)
data_source = (
DataSourceHandler().get_data_source(data_source_id)
DataSourceHandler().get_data_source(data_source_id, with_cache=True)
if data_source_id
else None
)

View file

@ -204,19 +204,23 @@ def get_builder_used_property_names(
BuilderWorkflowActionService,
)
# We query the data source first to populate the data source cache
data_sources = DataSourceService().get_builder_data_sources(
user, builder, with_cache=True
)
elements = list(ElementService().get_builder_elements(user, builder))
element_map = {e.id: e for e in elements}
element_results = get_element_property_names(elements, element_map)
ds_results = get_data_source_property_names(data_sources)
workflow_actions = BuilderWorkflowActionService().get_builder_workflow_actions(
user, builder
)
wa_results = get_workflow_action_property_names(workflow_actions, element_map)
data_sources = DataSourceService().get_builder_data_sources(user, builder)
ds_results = get_data_source_property_names(data_sources)
results = {
"internal": merge_dicts_no_duplicates(
wa_results["internal"], ds_results["internal"]

View file

@ -17,6 +17,9 @@ CACHE_KEY_PREFIX = "used_properties_for_page"
BUILDER_PREVIEW_USED_PROPERTIES_CACHE_TTL_SECONDS = 60
SENTINEL = "__no_results__"
class BuilderHandler:
def get_builder(self, builder_id: int) -> Builder:
"""
@ -87,11 +90,17 @@ class BuilderHandler:
(required only by the backend).
"""
return safe_get_or_set_cache(
def compute_properties():
properties = get_builder_used_property_names(user, builder)
return SENTINEL if properties is None else properties
result = safe_get_or_set_cache(
self.get_builder_used_properties_cache_key(user, builder),
self._get_builder_version_cache(builder),
default=lambda: get_builder_used_property_names(user, builder),
default=compute_properties,
timeout=settings.BUILDER_PUBLICLY_USED_PROPERTIES_CACHE_TTL_SECONDS
if builder.workspace_id
else BUILDER_PREVIEW_USED_PROPERTIES_CACHE_TTL_SECONDS,
)
return result if result != SENTINEL else None

View file

@ -0,0 +1,16 @@
# Generated by Django 5.0.9 on 2024-09-30 16:30
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("builder", "0050_page_query_params"),
]
operations = [
migrations.AlterModelOptions(
name="builderworkflowaction",
options={"ordering": ("order", "id")},
),
]

View file

@ -57,6 +57,9 @@ class BuilderWorkflowAction(
queryset = BuilderWorkflowAction.objects.filter(page=page, element=None)
return cls.get_highest_order_of_queryset(queryset) + 1
class Meta:
ordering = ("order", "id")
class NotificationWorkflowAction(BuilderWorkflowAction):
title = FormulaField(default="")

View file

@ -214,7 +214,7 @@ class LocalBaserowTableServiceFilterableMixin:
service_filter.value = new_formula
yield service_filter
def get_queryset(
def get_table_queryset(
self,
service: ServiceSubClass,
table: "Table",
@ -233,7 +233,7 @@ class LocalBaserowTableServiceFilterableMixin:
:return: the queryset with filters applied.
"""
queryset = super().get_queryset(service, table, dispatch_context, model)
queryset = super().get_table_queryset(service, table, dispatch_context, model)
queryset = self.get_dispatch_filters(service, queryset, model, dispatch_context)
dispatch_filters = dispatch_context.filters()
if dispatch_filters is not None and dispatch_context.is_publicly_filterable:
@ -254,15 +254,6 @@ class LocalBaserowTableServiceFilterableMixin:
queryset = adhoc_filters.apply_to_queryset(model, queryset)
return queryset
def enhance_queryset(self, queryset):
return (
super()
.enhance_queryset(queryset)
.prefetch_related(
"service_filters",
)
)
class LocalBaserowTableServiceSortableMixin:
"""
@ -356,7 +347,7 @@ class LocalBaserowTableServiceSortableMixin:
return sort_ordering, queryset
def get_queryset(
def get_table_queryset(
self,
service: ServiceSubClass,
table: "Table",
@ -375,7 +366,7 @@ class LocalBaserowTableServiceSortableMixin:
:return: the queryset with sortings applied.
"""
queryset = super().get_queryset(service, table, dispatch_context, model)
queryset = super().get_table_queryset(service, table, dispatch_context, model)
adhoc_sort = dispatch_context.sortings()
if adhoc_sort and dispatch_context.is_publicly_sortable:
@ -426,7 +417,7 @@ class LocalBaserowTableServiceSearchableMixin:
)
if isinstance(used_fields_from_parent, list) and service.search_query:
fields = [fo["field"] for fo in self.get_table_field_objects(service)]
fields = [fo["field"] for fo in self.get_table_field_objects(service) or []]
return used_fields_from_parent + [
f.tsv_db_column if SearchHandler.full_text_enabled() else f.db_column
for f in fields
@ -459,7 +450,7 @@ class LocalBaserowTableServiceSearchableMixin:
f"The `search_query` formula can't be resolved: {exc}"
) from exc
def get_queryset(
def get_table_queryset(
self,
service: ServiceSubClass,
table: "Table",
@ -478,7 +469,7 @@ class LocalBaserowTableServiceSearchableMixin:
:return: the queryset with the search query applied.
"""
queryset = super().get_queryset(service, table, dispatch_context, model)
queryset = super().get_table_queryset(service, table, dispatch_context, model)
search_mode = SearchHandler.get_default_search_mode_for_table(table)
service_search_query = self.get_dispatch_search(service, dispatch_context)
if service_search_query:

View file

@ -90,6 +90,7 @@ from baserow.contrib.integrations.local_baserow.utils import (
guess_cast_function_from_response_serializer_field,
guess_json_type_from_response_serializer_field,
)
from baserow.core.cache import local_cache
from baserow.core.formula import resolve_formula
from baserow.core.formula.registries import formula_runtime_function_registry
from baserow.core.handler import CoreHandler
@ -203,11 +204,10 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
if not model:
model = self.get_table_model(service)
queryset = self.get_queryset(service, table, dispatch_context, model)
queryset = self.get_table_queryset(service, table, dispatch_context, model)
return queryset
def get_queryset(
def get_table_queryset(
self,
service: ServiceSubClass,
table: "Table",
@ -225,7 +225,7 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
def enhance_queryset(self, queryset):
return queryset.select_related(
"table__database__workspace",
).prefetch_related("table__field_set")
)
def resolve_service_formulas(
self,
@ -482,8 +482,9 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
:return: A schema dictionary, or None if no `Table` has been applied.
"""
table = service.table
if not table:
field_objects = self.get_table_field_objects(service)
if field_objects is None:
return None
properties = {
@ -495,8 +496,7 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
"searchable": False,
}
}
for field_object in self.get_table_field_objects(service):
for field_object in field_objects:
# When a schema is being generated, we will exclude properties that the
# Application creator did not actively configure. A configured property
# is one that the Application is using in a formula, configuration
@ -506,10 +506,9 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
and field_object["name"] not in allowed_fields
):
continue
field_type = field_object["type"]
field = field_object["field"]
# Only `TextField` has a default value at the moment.
field = field_object["field"]
default_value = getattr(field, "text_default", None)
field_serializer = field_type.get_serializer(field, FieldSerializer)
properties[field.db_column] = {
@ -566,37 +565,41 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
Returns the model for the table associated with the given service.
"""
if getattr(service, "_table_model", None) is None:
table = service.table
if not service.table_id:
return None
if not table:
return None
return local_cache.get(
f"integration_service_{service.table_id}_table_model",
service.table.get_model,
)
setattr(service, "_table_model", table.get_model())
return getattr(service, "_table_model")
def get_table_field_objects(self, service: LocalBaserowTableService) -> List[Dict]:
def get_table_field_objects(
self, service: LocalBaserowTableService
) -> List[Dict] | None:
"""
Returns the fields of the table associated with the given service.
Returns the fields objects of the table of the given service.
:param service: The service we want the fields for.
:returns: The field objects from the table model.
"""
model = self.get_table_model(service)
if model is None:
return []
return None
return model.get_field_objects()
def get_context_data(
self, service: ServiceSubClass, allowed_fields: Optional[List[str]] = None
) -> Dict[str, Any]:
table = service.table
if not table:
field_objects = self.get_table_field_objects(service)
if field_objects is None:
return None
ret = {}
for field_object in self.get_table_field_objects(service):
for field_object in field_objects:
# When a context_data is being generated, we will exclude properties that
# the Application creator did not actively configure. A configured property
# is one that the Application is using in a formula, configuration
@ -619,22 +622,22 @@ class LocalBaserowTableServiceType(LocalBaserowServiceType):
def get_context_data_schema(
self, service: ServiceSubClass, allowed_fields: Optional[List[str]] = None
) -> Optional[Dict[str, Any]]:
table = service.table
if not table:
field_objects = self.get_table_field_objects(service)
if field_objects is None:
return None
properties = {}
fields = FieldHandler().get_fields(table, specific=True)
for field in fields:
if allowed_fields is not None and (field.db_column not in allowed_fields):
for field_object in field_objects:
if allowed_fields is not None and (
field_object["name"] not in allowed_fields
):
continue
field_type = field_type_registry.get_by_model(field)
if field_type.can_have_select_options:
properties[field.db_column] = {
if field_object["type"].can_have_select_options:
properties[field_object["name"]] = {
"type": "array",
"title": field.name,
"title": field_object["field"].name,
"default": None,
"items": {
"type": "object",
@ -703,7 +706,7 @@ class LocalBaserowViewServiceType(LocalBaserowTableServiceType):
return (
super()
.enhance_queryset(queryset)
.select_related("view")
.select_related("view__content_type")
.prefetch_related(
"view__viewfilter_set",
"view__filter_groups",
@ -1076,7 +1079,8 @@ class LocalBaserowListRowsUserServiceType(
# Maybe some fields were deleted in the meantime
# Let's check we still have them
available_fields = set(
[fo["name"] for fo in self.get_table_field_objects(service)] + ["id"]
[fo["name"] for fo in (self.get_table_field_objects(service) or [])]
+ ["id"]
)
# Ensure that only used fields are fetched from the database.

View file

@ -30,8 +30,6 @@ class AppAuthProviderHandler(BaseAuthProviderHandler):
base_queryset = cls.base_class.objects.filter(
user_source=user_source
).select_related(
"user_source",
"user_source__application",
"user_source__application__workspace",
)

View file

@ -0,0 +1,91 @@
from contextlib import contextmanager
from typing import Callable, TypeVar
from django.conf import settings
from asgiref.local import Local
T = TypeVar("T")
class LocalCache:
"""
A thread-safe and async-safe local cache implementation using asgiref.Local.
This cache provides request-scoped storage in Django applications via the
LocalCacheMiddleware. It ensures that data is isolated between different requests
and is automatically cleaned up after each request cycle, preventing data leakage
and maintaining proper cache lifecycle management.
"""
def __init__(self):
self._local = Local()
def get(self, key: str, default: T | Callable[[], T] = None) -> T:
"""
Get a value from the cache. If the key is not present:
- If the default is callable, call it to get the value.
- Otherwise, use the plain default value.
The computed or plain default value is stored in the cache.
As the cache is shared, ensure the key is unique to prevent collision.
"""
if not settings.BASEROW_USE_LOCAL_CACHE:
return default() if callable(default) else default
if not hasattr(self._local, "cache"):
self._local.cache = {}
cache = self._local.cache
if key not in cache:
value = default() if callable(default) else default
cache[key] = value
return cache[key]
def clear(self):
"""
Clear all data from the cache.
"""
if hasattr(self._local, "cache"):
del self._local.cache
@contextmanager
def context(self):
"""
Context manager for automatic cache lifecycle management. Clears the cache
before entering the context and ensures cleanup after exiting, even if an
exception occurs.
"""
self.clear()
try:
yield self
finally:
self.clear()
local_cache = LocalCache()
class LocalCacheMiddleware:
"""
Django middleware for managing the lifecycle of LocalCache.
This middleware ensures that the cache is cleared before and after
each request, preventing data leakage between requests and maintaining
proper cleanup.
Usage:
Add to MIDDLEWARE in Django settings:
'baserow.core.cache.LocalCacheMiddleware'
"""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
with local_cache.context():
return self.get_response(request)

View file

@ -81,7 +81,7 @@ def specific_iterator(
objects with the least amount of queries. If a queryset is provided respects the
annotations, select related and prefetch related of the provided query. This
function is only compatible with models having the `PolymorphicContentTypeMixin`
and `content_type property.`
and `content_type` property.
Can be used like:
@ -153,8 +153,8 @@ def specific_iterator(
if per_content_type_queryset_hook is not None:
objects = per_content_type_queryset_hook(model, objects)
for object in objects:
specific_objects[object.id] = object
for obj in objects:
specific_objects[obj.id] = obj
# Create an array with specific objects in the right order.
ordered_specific_objects = []

View file

@ -23,27 +23,37 @@ from .types import IntegrationForUpdate
class IntegrationHandler:
def get_integration(
self, integration_id: int, base_queryset: Optional[QuerySet] = None
self,
integration_id: int,
base_queryset: Optional[QuerySet] = None,
specific=True,
) -> Integration:
"""
Returns an integration instance from the database.
:param integration_id: The ID of the integration.
:param base_queryset: The base queryset use to build the query if provided.
:param specific: Whether we want the specific instance or not.
:raises IntegrationDoesNotExist: If the integration can't be found.
:return: The specific integration instance.
:return: The integration instance.
"""
queryset = (
base_queryset if base_queryset is not None else Integration.objects.all()
)
queryset = queryset.select_related("application__workspace")
try:
integration = (
queryset.select_related("application", "application__workspace")
.get(id=integration_id)
.specific
)
if specific:
integration = queryset.get(id=integration_id)
# We use the enhanced version of the queryset to get the related
# fields. This is also responsible for returning the specific instance.
integration = (
integration.get_type().get_queryset().get(id=integration_id)
)
else:
integration = queryset.get(id=integration_id)
except Integration.DoesNotExist:
raise IntegrationDoesNotExist()
@ -101,9 +111,7 @@ class IntegrationHandler:
integration_type = integration_type_registry.get_by_model(model)
return integration_type.enhance_queryset(queryset)
queryset = queryset.select_related(
"content_type", "application", "application__workspace"
)
queryset = queryset.select_related("application__workspace")
return specific_iterator(
queryset, per_content_type_queryset_hook=per_content_type_queryset_hook

View file

@ -3,6 +3,7 @@ from datetime import timedelta
from django.conf import settings
from baserow.config.celery import app
from baserow.core.cache import local_cache
from baserow.core.jobs.exceptions import JobCancelled
from baserow.core.jobs.registries import job_type_registry
from baserow.core.sentry import setup_user_in_sentry
@ -14,6 +15,7 @@ from baserow.core.telemetry.utils import setup_user_in_baggage_and_spans
queue="export",
soft_time_limit=settings.BASEROW_JOB_SOFT_TIME_LIMIT,
)
@local_cache.context()
def run_async_job(self, job_id: int):
"""Run the job task asynchronously"""

View file

@ -223,13 +223,13 @@ class PolymorphicContentTypeMixin:
"""Returns this instance in its most specific subclassed form."""
self._ensure_content_type_is_set()
content_type = ContentType.objects.get_for_id(self.content_type_id)
model_class = self.specific_class
if model_class is None:
return self
elif isinstance(self, model_class):
return self
else:
content_type = ContentType.objects.get_for_id(self.content_type_id)
return content_type.get_object_for_this_type(id=self.id)
@cached_property

View file

@ -1,7 +1,9 @@
from typing import List
from typing import Iterable, List
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AbstractUser
from baserow.core.cache import local_cache
from baserow.core.handler import CoreHandler
from baserow.core.integrations.operations import (
ListIntegrationsApplicationOperationType,
@ -192,22 +194,48 @@ class WorkspaceMemberOnlyPermissionManagerType(PermissionManagerType):
return getattr(actor, self.actor_cache_key, {}).get(workspace.id, False)
def get_user_ids_map_actually_in_workspace(
self,
workspace: Workspace,
users_to_query: Iterable[AbstractUser],
include_trash: bool = False,
):
"""
Return a `user_id -> is in the workspace` map. This version is cached for
a few seconds to prevent queries.
"""
cached = local_cache.get(f"workspace_user_{workspace.id}", dict)
missing = []
for user in users_to_query:
if user.id not in cached:
missing.append(user)
if missing:
user_ids_in_workspace = set(
CoreHandler()
.get_workspace_users(workspace, missing, include_trash=include_trash)
.values_list("user_id", flat=True)
)
for missing_user in missing:
cached[missing_user.id] = missing_user.id in user_ids_in_workspace
return cached
def check_multiple_permissions(self, checks, workspace=None, include_trash=False):
if workspace is None:
return {}
users_to_query = {c.actor for c in checks}
user_ids_in_workspace = set(
CoreHandler()
.get_workspace_users(workspace, users_to_query, include_trash=include_trash)
.values_list("user_id", flat=True)
user_id_map_in_workspace = self.get_user_ids_map_actually_in_workspace(
workspace, {c.actor for c in checks}, include_trash=include_trash
)
permission_by_check = {}
def check_workspace(actor):
return lambda: actor.id in user_ids_in_workspace
return lambda: user_id_map_in_workspace[actor.id]
for check in checks:
if self.is_actor_in_workspace(
@ -361,10 +389,9 @@ class StaffOnlySettingOperationPermissionManagerType(PermissionManagerType):
return always_allowed_operations, staff_only_operations
def check_multiple_permissions(self, checks, workspace=None, include_trash=False):
(
always_allowed_ops,
staff_only_ops,
) = self.get_permitted_operations_for_settings()
(always_allowed_ops, staff_only_ops) = local_cache.get(
"settings", self.get_permitted_operations_for_settings
)
result = {}
for check in checks:

View file

@ -275,6 +275,23 @@ class CustomFieldsInstanceMixin:
return None
def get_queryset(self):
"""
Returns the base queryset potentially enhanced by the `.enhance_queryset` method
for the model of this class. Requires the `ModelInstance` mixin.
"""
return self.enhance_queryset(self.model_class.objects.all())
def enhance_queryset(self, queryset):
"""
A hook to enhance the queryset for this type like adding `select_related`,
`prefetch_related` or stuff like that. Called by `.get_queryset` to generate
the base queryset to use for this type.
"""
return queryset
class PublicCustomFieldsInstanceMixin(CustomFieldsInstanceMixin):
"""

View file

@ -23,7 +23,7 @@ from .types import DispatchResult, ServiceForUpdate, UpdatedService
class ServiceHandler:
def get_service(
self, service_id: int, base_queryset: QuerySet[Service] = None
self, service_id: int, base_queryset: QuerySet[Service] = None, specific=True
) -> Service:
"""
Returns an service instance from the database.
@ -37,15 +37,23 @@ class ServiceHandler:
queryset = base_queryset if base_queryset is not None else Service.objects.all()
try:
service = (
queryset.select_related(
"integration",
"integration__application",
if specific:
service = queryset.get(id=service_id).specific
# We use the enhanced version of the queryset to get the related
# fields.
service = service.get_type().get_queryset().get(id=service_id)
if service.integration_id:
specific_integration = IntegrationHandler().get_integration(
service.integration_id, specific=True
)
service.__class__.integration.field.set_cached_value(
service, specific_integration
)
else:
service = queryset.select_related(
"integration__application__workspace",
)
.get(id=service_id)
.specific
)
).get(id=service_id)
except Service.DoesNotExist:
raise ServiceDoesNotExist()
@ -94,8 +102,6 @@ class ServiceHandler:
queryset = queryset.filter(integration=integration)
if specific:
queryset = queryset.select_related("content_type")
# Apply the type specific queryset enhancement for performance.
def per_content_type_queryset_hook(model, queryset):
service_type = service_type_registry.get_by_model(model)
@ -125,8 +131,8 @@ class ServiceHandler:
]
return specific_services
else:
queryset = queryset.select_related("integration__application")
return queryset
def create_service(self, service_type: ServiceType, **kwargs) -> Service:

View file

@ -11,6 +11,7 @@ from loguru import logger
from baserow.core.db import specific_iterator
from baserow.core.exceptions import ApplicationOperationNotSupported
from baserow.core.integrations.handler import IntegrationHandler
from baserow.core.models import Application
from baserow.core.registries import application_type_registry
from baserow.core.storage import ExportZipFile
@ -29,13 +30,17 @@ class UserSourceHandler:
allowed_fields_update = ["name", "integration"]
def get_user_source(
self, user_source_id: int, base_queryset: Optional[QuerySet] = None
self,
user_source_id: int,
base_queryset: Optional[QuerySet] = None,
specific=True,
) -> UserSource:
"""
Returns a user_source instance from the database.
:param user_source_id: The ID of the user_source.
:param base_queryset: The base queryset use to build the query if provided.
:param specific: To return the specific instances.
:raises UserSourceDoesNotExist: If the user_source can't be found.
:return: The specific user_source instance.
"""
@ -44,12 +49,23 @@ class UserSourceHandler:
base_queryset if base_queryset is not None else UserSource.objects.all()
)
queryset = queryset.select_related("application__workspace")
try:
user_source = (
queryset.select_related("application", "application__workspace")
.get(id=user_source_id)
.specific
)
if specific:
user_source = queryset.get(id=user_source_id).specific
if user_source.integration_id:
specific_integration = IntegrationHandler().get_integration(
user_source.integration_id, specific=True
)
user_source.__class__.integration.field.set_cached_value(
user_source, specific_integration
)
else:
user_source = queryset.select_related("integration").get(
id=user_source_id
)
except UserSource.DoesNotExist as exc:
raise UserSourceDoesNotExist() from exc
@ -59,12 +75,14 @@ class UserSourceHandler:
self,
user_source_uid: str,
base_queryset: Optional[QuerySet] = None,
specific=True,
) -> UserSource:
"""
Returns a user_source instance from the database.
:param user_source_uid: The uid of the user_source.
:param base_queryset: The base queryset use to build the query if provided.
:param specific: To return the specific instances.
:raises UserSourceDoesNotExist: If the user_source can't be found.
:return: The specific user_source instance.
"""
@ -73,12 +91,23 @@ class UserSourceHandler:
base_queryset if base_queryset is not None else UserSource.objects.all()
)
queryset = queryset.select_related("application__workspace")
try:
user_source = (
queryset.select_related("application", "application__workspace")
.get(uid=user_source_uid)
.specific
)
if specific:
user_source = queryset.get(uid=user_source_uid).specific
if user_source.integration_id:
specific_integration = IntegrationHandler().get_integration(
user_source.integration_id, specific=True
)
user_source.__class__.integration.field.set_cached_value(
user_source, specific_integration
)
else:
user_source = queryset.select_related("integration").get(
id=user_source_uid
)
except UserSource.DoesNotExist as exc:
raise UserSourceDoesNotExist() from exc
@ -136,9 +165,7 @@ class UserSourceHandler:
user_source_type = user_source_type_registry.get_by_model(model)
return user_source_type.enhance_queryset(queryset)
queryset = queryset.select_related(
"content_type", "application", "application__workspace"
)
queryset = queryset.select_related("application__workspace", "integration")
return specific_iterator(
queryset, per_content_type_queryset_hook=per_content_type_queryset_hook

View file

@ -40,3 +40,36 @@ class UserSourceFixtures:
)
user_sources.append(user_source)
return user_sources
def create_user_table_and_role(self, user, builder, user_role, integration=None):
"""Helper to create a User table with a particular user role."""
# Create the user table for the user_source
user_table, user_fields, user_rows = self.build_table(
user=user,
columns=[
("Email", "text"),
("Name", "text"),
("Password", "text"),
("Role", "text"),
],
rows=[
["foo@bar.com", "Foo User", "secret", user_role],
],
)
email_field, name_field, password_field, role_field = user_fields
integration = integration or self.create_local_baserow_integration(
user=user, application=builder
)
user_source = self.create_user_source(
user_source_type_registry.get("local_baserow").model_class,
application=builder,
integration=integration,
table=user_table,
email_field=email_field,
name_field=name_field,
role_field=role_field,
)
return user_source, integration

View file

@ -28,6 +28,7 @@ from rest_framework.test import APIRequestFactory
from sqlparse import format
from baserow.contrib.database.application_types import DatabaseApplicationType
from baserow.core.cache import local_cache
from baserow.core.context import clear_current_workspace_id
from baserow.core.exceptions import PermissionDenied
from baserow.core.jobs.registries import job_type_registry
@ -93,6 +94,14 @@ def api_request_factory():
return APIRequestFactory()
@pytest.fixture(autouse=True)
def reset_cache():
"""Automatically reset the short cache before each test."""
with local_cache.context():
yield
@pytest.fixture
def reset_schema(django_db_blocker):
yield

View file

@ -2,6 +2,7 @@ import json
from unittest.mock import ANY, MagicMock, patch
from django.db import transaction
from django.test import override_settings
from django.urls import reverse
import pytest
@ -18,10 +19,7 @@ from baserow.contrib.database.rows.handler import RowHandler
from baserow.contrib.database.views.models import SORT_ORDER_ASC
from baserow.core.services.models import Service
from baserow.core.user_sources.user_source_user import UserSourceUser
from baserow.test_utils.helpers import AnyInt, AnyStr
from tests.baserow.contrib.builder.api.user_sources.helpers import (
create_user_table_and_role,
)
from baserow.test_utils.helpers import AnyInt, AnyStr, setup_interesting_test_table
@pytest.fixture
@ -48,8 +46,7 @@ def data_source_fixture(data_fixture):
)
page = data_fixture.create_builder_page(user=user, builder=builder)
user_source, _ = create_user_table_and_role(
data_fixture,
user_source, _ = data_fixture.create_user_table_and_role(
user,
builder,
"foo_user_role",
@ -841,6 +838,7 @@ def test_dispatch_data_source_with_refinements_referencing_trashed_field(
service_filter = data_fixture.create_local_baserow_table_service_filter(
service=data_source.service, field=trashed_field, value="abc", order=0
)
url = reverse(
"api:builder:data_source:dispatch", kwargs={"data_source_id": data_source.id}
)
@ -872,6 +870,75 @@ def test_dispatch_data_source_with_refinements_referencing_trashed_field(
}
@pytest.mark.django_db
@pytest.mark.disabled_in_ci
# You must add --run-disabled-in-ci -s to pytest to run this test, you can do this in
# intellij by editing the run config for this test and adding --run-disabled-in-ci -s
# to additional args.
# pytest -k "test_dispatch_data_sources_perf" -s --run-disabled-in-ci
@override_settings(TESTS=False)
def test_dispatch_data_sources_perf(api_client, data_fixture, profiler):
user, token = data_fixture.create_user_and_token()
table1, _, _, _, _ = setup_interesting_test_table(data_fixture, user)
table2, _, _ = data_fixture.build_table(
user=user,
columns=[
("Name", "text"),
("My Color", "text"),
],
rows=[
[["2CV", "Green"]] * 40,
],
)
table3, _, _ = data_fixture.build_table(
user=user,
columns=[
("Name", "text"),
("My Color", "text"),
],
rows=[
[["Twingo", "White"]] * 40,
],
)
view = data_fixture.create_grid_view(user, table=table1)
builder = data_fixture.create_builder_application(user=user)
integration = data_fixture.create_local_baserow_integration(
user=user, application=builder
)
page = data_fixture.create_builder_page(user=user, builder=builder)
data_source1 = data_fixture.create_builder_local_baserow_list_rows_data_source(
user=user,
page=page,
integration=integration,
view=view,
table=table1,
)
data_source2 = data_fixture.create_builder_local_baserow_list_rows_data_source(
user=user,
page=page,
integration=integration,
table=table2,
)
data_source3 = data_fixture.create_builder_local_baserow_list_rows_data_source(
user=user,
page=page,
integration=integration,
table=table3,
)
url = reverse("api:builder:data_source:dispatch-all", kwargs={"page_id": page.id})
with profiler(html_report_name="data_sources_dispatch_perf"):
for _ in range(30):
api_client.post(
url,
{},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
@pytest.mark.django_db
def test_dispatch_data_source_with_adhoc_filters(api_client, data_fixture):
user, token = data_fixture.create_user_and_token()

View file

@ -5,7 +5,6 @@ from rest_framework.status import HTTP_200_OK
from baserow.contrib.builder.elements.models import Element
from baserow.contrib.builder.pages.models import Page
from baserow.core.user_sources.registries import user_source_type_registry
from baserow.core.user_sources.user_source_user import UserSourceUser
@ -90,44 +89,6 @@ def data_source_element_roles_fixture(data_fixture):
}
def create_user_table_and_role(
data_fixture, user, builder, user_role, integration=None
):
"""Helper to create a User table with a particular user role."""
# Create the user table for the user_source
user_table, user_fields, user_rows = data_fixture.build_table(
user=user,
columns=[
("Email", "text"),
("Name", "text"),
("Password", "text"),
("Role", "text"),
],
rows=[
["foo@bar.com", "Foo User", "secret", user_role],
],
)
email_field, name_field, password_field, role_field = user_fields
if integration is None:
integration = data_fixture.create_local_baserow_integration(
user=user, application=builder
)
user_source = data_fixture.create_user_source(
user_source_type_registry.get("local_baserow").model_class,
application=builder,
integration=integration,
table=user_table,
email_field=email_field,
name_field=name_field,
role_field=role_field,
)
return user_source, integration
@pytest.mark.django_db
def test_dispatch_data_sources_list_rows_no_elements(
api_client, data_fixture, data_source_fixture
@ -483,8 +444,7 @@ def test_dispatch_data_sources_list_rows_with_elements_and_role(
page = data_source_element_roles_fixture["page"]
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -633,8 +593,7 @@ def test_dispatch_data_sources_page_visibility_logged_in_allow_all_returns_eleme
integration = data_source_fixture["integration"]
user = data_source_fixture["user"]
user_source, _ = create_user_table_and_role(
data_fixture,
user_source, _ = data_fixture.create_user_table_and_role(
user,
data_source_fixture["builder_to"],
"foo_role",
@ -818,8 +777,7 @@ def test_dispatch_data_sources_page_visibility_logged_in_allow_all_except(
integration = data_source_fixture["integration"]
user = data_source_fixture["user"]
user_source, _ = create_user_table_and_role(
data_fixture,
user_source, _ = data_fixture.create_user_table_and_role(
user,
data_source_fixture["builder_to"],
user_role,

View file

@ -23,9 +23,6 @@ from baserow.contrib.database.views.models import SORT_ORDER_ASC
from baserow.core.exceptions import PermissionException
from baserow.core.services.exceptions import DoesNotExist, ServiceImproperlyConfigured
from baserow.core.user_sources.user_source_user import UserSourceUser
from tests.baserow.contrib.builder.api.user_sources.helpers import (
create_user_table_and_role,
)
@pytest.fixture
@ -768,8 +765,7 @@ def user_source_user_fixture(data_fixture):
)
page = data_fixture.create_builder_page(user=user, builder=builder)
user_source, _ = create_user_table_and_role(
data_fixture,
user_source, _ = data_fixture.create_user_table_and_role(
user,
builder,
"foo_user_role",
@ -1086,8 +1082,7 @@ def test_public_dispatch_data_sources_list_rows_with_elements_and_role(
page = data_source_element_roles_fixture["page"]
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -1270,8 +1265,7 @@ def test_public_dispatch_data_sources_list_rows_with_page_visibility_all(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -1451,8 +1445,7 @@ def test_public_dispatch_data_sources_get_row_with_page_visibility_all(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -1586,8 +1579,7 @@ def test_public_dispatch_data_sources_list_rows_with_page_visibility_logged_in(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -1743,8 +1735,7 @@ def test_public_dispatch_data_sources_get_row_with_page_visibility_logged_in(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -1839,8 +1830,7 @@ def test_list_elements_with_page_visibility_all(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,
@ -2013,8 +2003,7 @@ def test_list_elements_with_page_visibility_logged_in(
page.roles = page_roles
page.save()
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
data_source_element_roles_fixture["user"],
data_source_element_roles_fixture["builder_to"],
user_role,

View file

@ -1,37 +0,0 @@
from baserow.core.user_sources.registries import user_source_type_registry
def create_user_table_and_role(
data_fixture, user, builder, user_role, integration=None
):
"""Helper to create a User table with a particular user role."""
# Create the user table for the user_source
user_table, user_fields, user_rows = data_fixture.build_table(
user=user,
columns=[
("Email", "text"),
("Name", "text"),
("Password", "text"),
("Role", "text"),
],
rows=[
["foo@bar.com", "Foo User", "secret", user_role],
],
)
email_field, name_field, password_field, role_field = user_fields
integration = integration or data_fixture.create_local_baserow_integration(
user=user, application=builder
)
user_source = data_fixture.create_user_source(
user_source_type_registry.get("local_baserow").model_class,
application=builder,
integration=integration,
table=user_table,
email_field=email_field,
name_field=name_field,
role_field=role_field,
)
return user_source, integration

View file

@ -1338,7 +1338,7 @@ def test_data_source_data_extract_properties_calls_correct_service_type(
result = DataSourceDataProviderType().extract_properties(path)
assert result == {mocked_data_source.service_id: expected}
mocked_get_data_source.assert_called_once_with(int(data_source_id))
mocked_get_data_source.assert_called_once_with(int(data_source_id), with_cache=True)
mocked_service_type.extract_properties.assert_called_once_with([expected])
@ -1434,7 +1434,7 @@ def test_data_source_context_extract_properties_calls_correct_service_type(
result = DataSourceContextDataProviderType().extract_properties(path)
assert result == {mocked_data_source.service_id: expected}
mocked_get_data_source.assert_called_once_with(int(data_source_id))
mocked_get_data_source.assert_called_once_with(int(data_source_id), with_cache=True)
mocked_service_type.extract_properties.assert_called_once_with([expected])
@ -1512,7 +1512,7 @@ def test_data_source_context_data_provider_extract_properties_raises_if_data_sou
with pytest.raises(InvalidBaserowFormula):
DataSourceContextDataProviderType().extract_properties(path)
mock_get_data_source.assert_called_once_with(int(path[0]))
mock_get_data_source.assert_called_once_with(int(path[0]), with_cache=True)
@pytest.mark.parametrize(
@ -1539,7 +1539,7 @@ def test_data_source_data_provider_extract_properties_raises_if_data_source_does
with pytest.raises(InvalidBaserowFormula):
DataSourceDataProviderType().extract_properties(path)
mock_get_data_source.assert_called_once_with(int(path[0]))
mock_get_data_source.assert_called_once_with(int(path[0]), with_cache=True)
@pytest.mark.parametrize("path", ([], [""], ["foo"]))
@ -1581,7 +1581,9 @@ def test_current_record_extract_properties_raises_if_data_source_doesnt_exist(
with pytest.raises(InvalidBaserowFormula):
CurrentRecordDataProviderType().extract_properties(path, invalid_data_source_id)
mock_get_data_source.assert_called_once_with(invalid_data_source_id)
mock_get_data_source.assert_called_once_with(
invalid_data_source_id, with_cache=True
)
@pytest.mark.django_db
@ -1617,7 +1619,7 @@ def test_current_record_extract_properties_calls_correct_service_type(
result = CurrentRecordDataProviderType().extract_properties(path, fake_element_id)
assert result == {mocked_data_source.service_id: expected_field}
mock_get_data_source.assert_called_once_with(fake_element_id)
mock_get_data_source.assert_called_once_with(fake_element_id, with_cache=True)
mocked_service_type.extract_properties.assert_called_once_with(
["0", expected_field]
)
@ -1676,7 +1678,7 @@ def test_current_record_extract_properties_called_with_correct_path(
schema_property,
)
mock_get_data_source.assert_called_once_with(data_source_id)
mock_get_data_source.assert_called_once_with(data_source_id, with_cache=True)
if returns_list:
if schema_property:

View file

@ -20,9 +20,6 @@ from baserow.core.exceptions import CannotCalculateIntermediateOrder
from baserow.core.services.registries import service_type_registry
from baserow.core.user_sources.user_source_user import UserSourceUser
from baserow.test_utils.helpers import AnyStr
from tests.baserow.contrib.builder.api.user_sources.helpers import (
create_user_table_and_role,
)
@pytest.mark.django_db
@ -525,8 +522,7 @@ def test_dispatch_data_source_doesnt_return_formula_field_names(
)
page = data_fixture.create_builder_page(user=user, builder=builder)
user_source, _ = create_user_table_and_role(
data_fixture,
user_source, _ = data_fixture.create_user_table_and_role(
user,
builder,
"foo_user_role",

View file

@ -14,9 +14,6 @@ from baserow.contrib.builder.data_sources.exceptions import (
from baserow.contrib.builder.elements.element_types import collection_element_types
from baserow.core.services.utils import ServiceAdhocRefinements
from baserow.core.user_sources.user_source_user import UserSourceUser
from tests.baserow.contrib.builder.api.user_sources.helpers import (
create_user_table_and_role,
)
def test_dispatch_context_page_range():
@ -48,8 +45,7 @@ def test_dispatch_context_page_from_context(mock_get_field_names, data_fixture):
user = data_fixture.create_user()
page = data_fixture.create_builder_page(user=user)
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
user,
page.builder,
"foo_user_role",
@ -351,8 +347,7 @@ def test_builder_dispatch_context_public_allowed_properties_is_cached(
)
builder = data_fixture.create_builder_application(user=user)
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
user,
builder,
"foo_user_role",
@ -393,7 +388,7 @@ def test_builder_dispatch_context_public_allowed_properties_is_cached(
}
# Initially calling the property should cause a bunch of DB queries.
with django_assert_num_queries(12):
with django_assert_num_queries(9):
result = dispatch_context.public_allowed_properties
assert result == expected_results

View file

@ -557,11 +557,12 @@ def test_duplicate_element_with_workflow_action(data_fixture):
def test_get_element_workflow_actions(data_fixture):
page = data_fixture.create_builder_page()
element = data_fixture.create_builder_button_element()
workflow_action_one = data_fixture.create_notification_workflow_action(
page=page, element=element
)
# Order is intentionally switched to check that the result is ordered correctly
workflow_action_two = data_fixture.create_notification_workflow_action(
page=page, element=element
page=page, element=element, order=2
)
workflow_action_one = data_fixture.create_notification_workflow_action(
page=page, element=element, order=1
)
[

View file

@ -7,9 +7,6 @@ import pytest
from baserow.contrib.builder.handler import CACHE_KEY_PREFIX, BuilderHandler
from baserow.core.exceptions import ApplicationDoesNotExist
from baserow.core.user_sources.user_source_user import UserSourceUser
from tests.baserow.contrib.builder.api.user_sources.helpers import (
create_user_table_and_role,
)
User = get_user_model()
@ -111,8 +108,7 @@ def test_public_allowed_properties_is_cached(data_fixture, django_assert_num_que
)
builder = data_fixture.create_builder_application(user=user)
user_source, integration = create_user_table_and_role(
data_fixture,
user_source, integration = data_fixture.create_user_table_and_role(
user,
builder,
"foo_user_role",
@ -146,7 +142,7 @@ def test_public_allowed_properties_is_cached(data_fixture, django_assert_num_que
}
# Initially calling the property should cause a bunch of DB queries.
with django_assert_num_queries(12):
with django_assert_num_queries(9):
result = handler.get_builder_public_properties(user_source_user, builder)
assert result == expected_results

View file

@ -45,6 +45,7 @@ from baserow.contrib.database.rows.handler import RowHandler
from baserow.contrib.database.table.handler import TableHandler
from baserow.contrib.database.table.models import GeneratedTableModel, Table
from baserow.contrib.database.views.handler import ViewHandler
from baserow.core.cache import local_cache
from baserow.core.handler import CoreHandler
from baserow.core.models import TrashEntry, WorkspaceUser
from baserow.core.registries import ImportExportConfig
@ -2206,10 +2207,10 @@ def test_clear_link_row_limit_selection_view_when_view_is_deleted(
view_handler = ViewHandler()
with CaptureQueriesContext(connection) as queries_request_1:
with CaptureQueriesContext(connection) as queries_request_1, local_cache.context():
view_handler.delete_view(user, view_3)
with CaptureQueriesContext(connection) as queries_request_2:
with CaptureQueriesContext(connection) as queries_request_2, local_cache.context():
view_handler.delete_view(user, view)
assert len(queries_request_1.captured_queries) + 1 == len(

View file

@ -34,7 +34,7 @@ def get_test_service_type(mixin_class):
@pytest.mark.django_db
def test_local_baserow_table_service_filterable_mixin_get_queryset(
def test_local_baserow_table_service_filterable_mixin_get_table_queryset(
data_fixture,
):
service_type = get_test_service_type(LocalBaserowTableServiceFilterableMixin)
@ -60,7 +60,7 @@ def test_local_baserow_table_service_filterable_mixin_get_queryset(
# No filters of any kind.
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alessia.id, alex.id, alastair.id, alexandra.id]
@ -77,7 +77,7 @@ def test_local_baserow_table_service_filterable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alessia.id, alex.id, alexandra.id]
@ -100,7 +100,7 @@ def test_local_baserow_table_service_filterable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alexandra.id]
@ -244,7 +244,7 @@ def test_local_baserow_table_service_filterable_mixin_get_dispatch_filters_raise
@pytest.mark.django_db
def test_local_baserow_table_service_sortable_mixin_get_queryset(
def test_local_baserow_table_service_sortable_mixin_get_table_queryset(
data_fixture,
):
service_type = get_test_service_type(LocalBaserowTableServiceSortableMixin)
@ -270,7 +270,7 @@ def test_local_baserow_table_service_sortable_mixin_get_queryset(
# No sorts of any kind.
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [aardvark.id, badger.id, crow.id, dragonfly.id]
@ -282,7 +282,7 @@ def test_local_baserow_table_service_sortable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [dragonfly.id, crow.id, badger.id, aardvark.id]
@ -294,7 +294,7 @@ def test_local_baserow_table_service_sortable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [aardvark.id, badger.id, crow.id, dragonfly.id]
@ -347,7 +347,7 @@ def test_local_baserow_table_service_sortable_mixin_get_dispatch_sorts_raises_ex
@pytest.mark.django_db(transaction=True)
def test_local_baserow_table_service_searchable_mixin_get_queryset(
def test_local_baserow_table_service_searchable_mixin_get_table_queryset(
data_fixture,
):
service_type = get_test_service_type(LocalBaserowTableServiceSearchableMixin)
@ -374,7 +374,7 @@ def test_local_baserow_table_service_searchable_mixin_get_queryset(
# No search query of any kind.
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alessia.id, alex.id, alastair.id, alexandra.id]
@ -384,7 +384,7 @@ def test_local_baserow_table_service_searchable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alessia.id, alex.id, alexandra.id]
@ -396,7 +396,7 @@ def test_local_baserow_table_service_searchable_mixin_get_queryset(
assert [
row.id
for row in service_type.get_queryset(
for row in service_type.get_table_queryset(
service, table, dispatch_context, table_model
)
] == [alexandra.id]

View file

@ -9,6 +9,7 @@ import pytest
from baserow.contrib.database.models import Database
from baserow.contrib.database.operations import ListTablesDatabaseTableOperationType
from baserow.core.cache import local_cache
from baserow.core.exceptions import (
PermissionDenied,
UserInvalidWorkspacePermissionsError,
@ -1141,7 +1142,7 @@ def test_allow_if_template_permission_manager_filter_queryset(data_fixture):
"member",
"token",
"basic",
]
],
)
def test_allow_if_template_permission_manager_query_count(data_fixture):
buser = data_fixture.create_user(username="Auth user")
@ -1161,7 +1162,9 @@ def test_allow_if_template_permission_manager_query_count(data_fixture):
workspace=workspace_1,
)
with CaptureQueriesContext(connection) as query_not_for_template:
with CaptureQueriesContext(
connection
) as query_not_for_template, local_cache.context():
CoreHandler().check_permissions(
buser,
UpdateIntegrationOperationType.type,

View file

@ -0,0 +1,80 @@
from django.http import HttpRequest
from django.test import override_settings
import pytest
from baserow.core.cache import LocalCacheMiddleware, local_cache
# Simulate a Django view
def mock_view(request):
user_profile = local_cache.get(
"user_profile", lambda: {"id": 1, "name": "Test User"}
)
return user_profile
@pytest.fixture
def middleware():
"""Provide an instance of the middleware."""
return LocalCacheMiddleware(get_response=mock_view)
def test_cache_storage(middleware):
"""Test that the cache stores and retrieves values correctly."""
request = HttpRequest()
response = middleware(request)
assert response == {"id": 1, "name": "Test User"}
# Test that the value is cached
cached_value = local_cache.get("user_profile")
assert cached_value is None
def test_callable_default():
"""Test that callable defaults are executed and cached."""
# Check if the callable default was executed
assert local_cache.get("user_profile", lambda: "test") == "test"
def test_cache_isolation(middleware):
"""Test that the cache is isolated between simulated requests."""
local_cache.get("user_profile", "before")
request1 = HttpRequest()
result = middleware(request1)
assert result == {"id": 1, "name": "Test User"}
assert local_cache.get("user_profile", "No Cache") == "No Cache"
# Simulate a new request and ensure the cache is isolated
request2 = HttpRequest()
middleware(request2)
# Ensure the second request starts with an empty cache
assert local_cache.get("user_profile", "No Cache") == "No Cache"
def test_cache_cleanup_after_request(middleware):
"""Test that the cache is cleared after the request lifecycle."""
request = HttpRequest()
middleware(request)
# and after the cache, the cache should be cleaned up
assert local_cache.get("user_profile", "after") == "after"
@override_settings(BASEROW_USE_LOCAL_CACHE=False)
def test_cache_disabled():
"""
Test that the cache does not store values when BASEROW_USE_LOCAL_CACHE is False.
"""
assert local_cache.get("user_profile", lambda: "disabled") == "disabled"
assert local_cache.get("user_profile") is None

View file

@ -0,0 +1,7 @@
{
"type": "refactor",
"message": "[Builder] Improve data dispatching performances",
"issue_number": 2978,
"bullet_points": [],
"created_at": "2024-09-26"
}

View file

@ -199,7 +199,7 @@ x-backend-variables: &backend-variables
BASEROW_MAX_HEALTHY_CELERY_QUEUE_SIZE:
BASEROW_ENTERPRISE_PERIODIC_DATA_SYNC_CHECK_INTERVAL_MINUTES:
BASEROW_ENTERPRISE_MAX_PERIODIC_DATA_SYNC_CONSECUTIVE_ERRORS:
BASEROW_USE_LOCAL_CACHE:
services:
# A caddy reverse proxy sitting in-front of all the services. Responsible for routing

View file

@ -199,6 +199,17 @@ class LocalBaserowPasswordAppAuthProviderType(AppAuthProviderType):
**kwargs,
)
def get_user_model_field_ids(self, auth_provider: AuthProviderModelSubClass):
"""
This method is specific to local_baserow app_auth_providers to return the list
of fields used by this provider to select them on the table model to save
queries.
"""
return (
[auth_provider.password_field_id] if auth_provider.password_field_id else []
)
def authenticate(
self,
auth_provider: AuthProviderModelSubClass,
@ -213,8 +224,9 @@ class LocalBaserowPasswordAppAuthProviderType(AppAuthProviderType):
user = user_source.get_type().get_user(user_source, email=email)
password_field = auth_provider.password_field
encoded_password = getattr(user.original_user, password_field.db_column)
encoded_password = getattr(
user.original_user, f"field_{auth_provider.password_field_id}"
)
if encoded_password and check_password(password, encoded_password):
return user

View file

@ -446,7 +446,30 @@ class LocalBaserowUserSourceType(UserSourceType):
integration = user_source.integration.specific
model = table.get_model()
app_auth_providers = (
AppAuthProviderHandler.list_app_auth_providers_for_user_source(user_source)
)
providers_fields = [
f
for ap in app_auth_providers
if hasattr(ap.get_type(), "get_user_model_field_ids")
for f in ap.get_type().get_user_model_field_ids(ap)
]
model = table.get_model(
add_dependencies=False,
field_ids=[
f
for f in [
user_source.email_field_id,
user_source.name_field_id,
user_source.role_field_id,
*providers_fields,
]
if f
],
)
CoreHandler().check_permissions(
integration.authorized_user,
@ -614,8 +637,8 @@ class LocalBaserowUserSourceType(UserSourceType):
user_source,
user,
user.id,
getattr(user, user_source.name_field.db_column),
getattr(user, user_source.email_field.db_column),
getattr(user, f"field_{user_source.name_field_id}"),
getattr(user, f"field_{user_source.email_field_id}"),
self.get_user_role(user, user_source),
)

View file

@ -1,5 +1,5 @@
from collections import defaultdict
from functools import cached_property
from functools import cached_property, partial
from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict
from django.contrib.auth import get_user_model
@ -7,6 +7,7 @@ from django.contrib.auth.models import AbstractUser
from baserow_premium.license.handler import LicenseHandler
from baserow.core.cache import local_cache
from baserow.core.exceptions import PermissionDenied
from baserow.core.models import Workspace
from baserow.core.registries import (
@ -43,7 +44,10 @@ class RolePermissionManagerType(PermissionManagerType):
:param workspace: The workspace in which we want to use this permission manager.
"""
return LicenseHandler.workspace_has_feature(RBAC, workspace)
return local_cache.get(
f"has_rbac_permission_{workspace.id}",
partial(LicenseHandler.workspace_has_feature, RBAC, workspace),
)
def get_role_operations(self, role: Role) -> List[str]:
"""

View file

@ -1,3 +1,4 @@
from functools import partial
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from django.contrib.auth import get_user_model
@ -48,6 +49,7 @@ from baserow.contrib.database.views.operations import (
UpdateViewSlugOperationType,
UpdateViewSortOperationType,
)
from baserow.core.cache import local_cache
from baserow.core.exceptions import PermissionDenied, PermissionException
from baserow.core.handler import CoreHandler
from baserow.core.registries import PermissionManagerType, object_scope_type_registry
@ -172,7 +174,10 @@ class ViewOwnershipPermissionManagerType(PermissionManagerType):
):
continue
premium = LicenseHandler.user_has_feature(PREMIUM, actor, workspace)
premium = local_cache.get(
f"has_premium_permission_{actor.id}_{workspace.id}",
partial(LicenseHandler.user_has_feature, PREMIUM, actor, workspace),
)
view_scope_type = object_scope_type_registry.get("database_view")
view = object_scope_type_registry.get_parent(
@ -259,7 +264,10 @@ class ViewOwnershipPermissionManagerType(PermissionManagerType):
if not workspace:
return queryset
premium = LicenseHandler.user_has_feature(PREMIUM, actor, workspace)
premium = local_cache.get(
f"has_premium_permission_{actor.id}_{workspace.id}",
lambda: LicenseHandler.user_has_feature(PREMIUM, actor, workspace),
)
if premium:
allowed_tables = CoreHandler().filter_queryset(

View file

@ -10,7 +10,7 @@ const populateElement = (element, registry) => {
element._ = {
contentLoading: true,
content: [],
hasNextPage: false,
hasNextPage: true,
reset: 0,
shouldBeFocused: false,
elementNamespacePath: null,

View file

@ -44,7 +44,7 @@ const mutations = {
CLEAR_CONTENT(state, { element }) {
element._.content = []
element._.hasNextPage = false
element._.hasNextPage = true
},
TRIGGER_RESET(state, { element }) {
element._.reset += 1
@ -106,7 +106,11 @@ const actions = {
* - Root collection element (with a dataSource):
* - Parent collection element (this `element`!) with a schema property.
*/
if (dataSource === null) {
if (!dataSource) {
// We clearly can't have more page for that one
commit('SET_HAS_MORE_PAGE', { element, value: false })
commit('SET_LOADING', { element, value: false })
if (!element.schema_property) {
// We have a collection element that supports schema properties, and
// we have A) no data source and B) no schema property
@ -117,8 +121,6 @@ const actions = {
return
}
commit('SET_LOADING', { element, value: true })
// Collect all collection element ancestors, with a `data_source_id`.
const collectionAncestors = this.app.store.getters[
'element/getAncestors'
@ -167,6 +169,8 @@ const actions = {
element,
value: elementContent,
})
// No more content for sure
commit('SET_HAS_MORE_PAGE', { element, value: false })
commit('SET_LOADING', { element, value: false })
return
}
@ -198,7 +202,8 @@ const actions = {
])
// Everything is already loaded we can quit now
if (!rangeToFetch) {
if (!rangeToFetch || !getters.getHasMorePage(element)) {
commit('SET_LOADING', { element, value: false })
return
}
rangeToFetch = [rangeToFetch[0], rangeToFetch[1] - rangeToFetch[0]]

View file

@ -189,7 +189,6 @@ exports[`RecordSelectorElement does not paginate if API returns 400/404 1`] = `
<div
class="infinite-scroll__loading-wrapper"
style=""
>
<!---->
</div>
@ -406,7 +405,6 @@ exports[`RecordSelectorElement does not paginate if API returns 400/404 2`] = `
<div
class="infinite-scroll__loading-wrapper"
style=""
>
<!---->
</div>
@ -623,7 +621,6 @@ exports[`RecordSelectorElement does not paginate if API returns 400/404 3`] = `
<div
class="infinite-scroll__loading-wrapper"
style=""
>
<!---->
</div>