diff --git a/.env.sample b/.env.sample index 21422dc8bd9..7858837882e 100644 --- a/.env.sample +++ b/.env.sample @@ -248,4 +248,7 @@ DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 # FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=False \ No newline at end of file +XLSX_UPLOAD_ENABLED=False + +# List of trusted hosts (format: domain:port) allowed to bypass URL safety validation. +# SAFE_URL_TRUSTED_HOSTS=[] diff --git a/.env_dev b/.env_dev index 9c9b7eaa176..755e6c3b39a 100644 --- a/.env_dev +++ b/.env_dev @@ -210,4 +210,4 @@ UPSERT_CHUNK_SIZE= 100 UPSERT_LIMIT_ERROR_LOG=100 # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=False \ No newline at end of file +XLSX_UPLOAD_ENABLED=False diff --git a/SECURITY.md b/SECURITY.md index f24bd36a3ab..273f5de3c0f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,17 +2,42 @@ ## Supported Versions -GeoNode versions supported with security updates. +GeoNode [versions](https://github.com/GeoNode/geonode/wiki/Releases) supported with security updates. | Version | Supported | | ------- | ------------------ | -| 4.0.x | :white_check_mark: | +| 5.1.x | :white_check_mark: | | 5.0.x | :white_check_mark: | -| < 4.0 | :x: | +| < 5 | :x: | + +This approach provides ample time for upgrading ensuring you are always working with a supported GeoNode release. + +If your organization is making use of a GeoNode version that is no longer in use by the community all is not lost. +You can volunteer on the developer list to make additional releases, or engage with one of our +[Commercial Support](https://geonode.org/providers/) providers. ## Reporting a Vulnerability -- **DO NOT** send a security alert on the public mailing list or any other public channel -- **SEND** the report to geonode-psc@lists.osgeo.org and be prapred to collaborate with the GeoNode PSC +1. **DO NOT** report vulnerabilities on the public mailing list or any other public channel +2. There are **two options** to report a security vulnerability: + - Send the report by email to geonode-psc@lists.osgeo.org and be prepared to collaborate with the GeoNode PSC + - Navigate to [Private vulnerability reporting](https://github.com/geonode/geonode/security/advisories/new) and create a new vulnerability report. For more information see [GitHub documentation](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability#privately-reporting-a-security-vulnerability). + +3. There is no expected response time. +4. Keep in mind participants are volunteering their time, an extensive fix may require fundraising/resources. Thanks for you support and help! + +## Coordinated vulnerability disclosure + +Disclosure policy: + +1. The reported vulnerability has been verified by working with the GeoNode PSC +2. GitHub [security advisory](https://github.com/geonode/geonode/security) is used to reserve a CVE number by the GeoNode Organization +3. A fix or documentation clarification is accepted and backported to active branches +4. A fix is included for the active branches release downloads ([reelases](https://github.com/GeoNode/geonode/releases), or issued via emergency update) +6. The CVE vulnerability is published by the GeoNode Organization with mitigation and patch instructions + +This represents a balance between transparency and participation that does not overwhelm participants. +Those seeking greater visibility are encouraged to volunteer with the [geonode-devel](https://lists.osgeo.org/cgi-bin/mailman/listinfo/geonode-devel) list; +or work with one of the [commercial support providers](https://geonode.org/providers/) who participate on behalf of their customers. diff --git a/celery-cmd b/celery-cmd index bb2f04529e5..4872be0e2c6 100644 --- a/celery-cmd +++ b/celery-cmd @@ -16,13 +16,14 @@ CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"} # Celery beat settings CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"} +CELERY__BEAT_DB=${CELERY__BEAT_DB:-"/mnt/volumes/statics/celerybeat-schedule"} CELERY__BEAT_LOG=${CELERY__BEAT_LOG:-"/var/log/celery_beat.log"} # Harvester settings CELERY__HARVESTER_WORKER_NAME=${CELERY__HARVESTER_WORKER_NAME:-"harvesting_worker@%h"} CELERY__HARVESTER_CONCURRENCY=${CELERY__HARVESTER_CONCURRENCY:-"10"} CELERY__HARVESTER_AUTOSCALE_VALUES=${CELERY__HARVESTER_AUTOSCALE_VALUES:-"15,10"} -CELERY__HARVESTER_MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"500000"} +CELERY__HARVESTER_MAX_MEMORY_PER_CHILD=${CELERY__HARVESTER_MAX_MEMORY_PER_CHILD:-"500000"} # --- FIX: Remove stale Beat pidfile before starting beat --- BEAT_PIDFILE="/tmp/celerybeat.pid" @@ -43,6 +44,7 @@ fi echo "Starting Celery Beat..." $CELERY_BIN -A $CELERY_APP beat --scheduler=$CELERY__BEAT_SCHEDULE \ + --schedule=$CELERY__BEAT_DB \ --loglevel=$CELERY__LOG_LEVEL -f $CELERY__BEAT_LOG --pidfile=/tmp/celerybeat.pid & echo "Starting Default Celery Worker..." @@ -67,4 +69,4 @@ wait -n # Exit with the status of the process that exited first # Docker will restart the container if this is non-zero (i.e., a failure) -exit $? \ No newline at end of file +exit $? diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 90b46ad7bae..25da411312f 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -52,7 +52,7 @@ services: # Nginx is serving django static and media files and proxies to django and geonode geonode: - image: geonode/nginx:1.28.0-v1 + image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - .env @@ -79,7 +79,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.28.x-latest + image: geonode/geoserver:2.28.4-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -105,7 +105,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest + image: geonode/geoserver_data:2.28.4-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml index b0fa460421c..c62683831e9 100644 --- a/docker-compose-geoserver-server.yml +++ b/docker-compose-geoserver-server.yml @@ -2,7 +2,7 @@ version: '2.2' services: data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest + image: geonode/geoserver_data:2.28.4-latest restart: on-failure container_name: gsconf4${COMPOSE_PROJECT_NAME} labels: @@ -13,7 +13,7 @@ services: - geoserver-data-dir:/geoserver_data/data geoserver: - image: geonode/geoserver:2.28.x-latest + image: geonode/geoserver:2.28.4-latest restart: unless-stopped container_name: geoserver4${COMPOSE_PROJECT_NAME} stdin_open: true diff --git a/docker-compose-test.yml b/docker-compose-test.yml index a5217585bc8..5ac1b1da494 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -40,7 +40,7 @@ services: # Nginx is serving django static and media files and proxies to django and geonode geonode: - image: geonode/nginx:1.28.0-v1 + image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - .env_test @@ -80,7 +80,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.28.x-latest + image: geonode/geoserver:2.28.4-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -106,7 +106,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest + image: geonode/geoserver_data:2.28.4-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker-compose.yml b/docker-compose.yml index d583ce337de..16480bfe4fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: # Nginx is serving django static and media files and proxies to django and geonode geonode: - image: geonode/nginx:1.28.0-v1 + image: geonode/nginx:1.31.6-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - .env @@ -92,7 +92,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.28.x-latest + image: geonode/geoserver:2.28.5-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -118,7 +118,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest + image: geonode/geoserver_data:2.28.5-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker/base/ubuntu/Dockerfile b/docker/base/ubuntu/Dockerfile index d34907376f4..01fca205f06 100644 --- a/docker/base/ubuntu/Dockerfile +++ b/docker/base/ubuntu/Dockerfile @@ -6,6 +6,8 @@ RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-k # will install python3.10 RUN apt-get install lsb-release -y RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee /etc/apt/sources.list.d/pgdg.list +# Keep GDAL from the Ubuntu archive, pgdg ships newer builds that break libgdal-dev / pip GDAL==3.8.4 +RUN printf 'Package: *gdal*\nPin: release o=apt.postgresql.org\nPin-Priority: -1\n' > /etc/apt/preferences.d/pgdg-no-gdal # Prepraing dependencies RUN apt-get install -y \ diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8121b37edda..f8eed73595a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -63,12 +63,12 @@ nav: - Attributes Table: "user-guide/maps/maps_configuration/attribute_table.md" - Timeline: "user-guide/maps/maps_configuration/timeline.md" - Other Menu Tools: "user-guide/maps/maps_configuration/options_menu.md" - - Documents: - - Upload/Add Documents: "user-guide/documents/upload.md" + - Documents: "user-guide/documents/upload.md" - GeoStories: "user-guide/geostory/geostory.md" - Dashboards: "user-guide/dashboard/dashboard.md" - Styling: "user-guide/resource_styling.md" - Sharing: "user-guide/sharing.md" + - Default Language: "user-guide/default-language.md" - Metadata: "user-guide/metadata.md" - Administration: - GeoNode Admins Guide: @@ -104,7 +104,6 @@ nav: - Customize the look and feel: - GeoNode Themes: "admin/gnode_theme/geonode_themes.md" - Theming your GeoNode Project: "admin/gnode_theme/theme_geonode_project.md" - - Changing the Default Language: "admin/default_lang.md" - Thesauri: "admin/thesauri/thesauri.md" - Developer Guide: - API Usage: "development/api.md" diff --git a/docs/src/admin/admin_panel/admin_panel_theming.md b/docs/src/admin/admin_panel/admin_panel_theming.md index f9edaf45041..b19f989839c 100644 --- a/docs/src/admin/admin_panel/admin_panel_theming.md +++ b/docs/src/admin/admin_panel/admin_panel_theming.md @@ -30,8 +30,6 @@ Just below the `Description` field, you will find the `Enabled` checkbox, which *Theme Name and Description* /// -## Jumbotron and Get Started link - !!! Note Remember, every time you want to apply some changes to the theme, you **must** save the theme and reload the GeoNode browser tab. @@ -39,6 +37,8 @@ Just below the `Description` field, you will find the `Enabled` checkbox, which ![](img/theming/view_site.png){ align=center } +## Jumbotron and Get Started link + The next section allows you to define the first important theme properties. This part involves the GeoNode main page sections. ![](img/theming/theme_properties.png){ align=center } diff --git a/docs/src/admin/admin_panel/group.md b/docs/src/admin/admin_panel/group.md index 10c2cbea526..37738a241ec 100644 --- a/docs/src/admin/admin_panel/group.md +++ b/docs/src/admin/admin_panel/group.md @@ -1,7 +1,7 @@ ## Creating a Group -| In GeoNode is possible to create new groups with set of permissions which will be inherited by all the group members. -| The creation of a Group can be done both on the GeoNode UI and on the *Admin Panel*, we will explain how in this paragraph. +In GeoNode is possible to create new groups with set of permissions which will be inherited by all the group members. +The creation of a Group can be done both on the GeoNode UI and on the *Admin Panel*, we will explain how in this paragraph. The `Create Groups` link of *About* menu in the navigation bar allows administrators to reach the *Group Creation Page*. @@ -26,9 +26,9 @@ The new created group will be searchable in the *Groups List Page*. !!! note "Important notes" The `Create a New Group` button on the *Groups List Page* allows to reach the *Group Creation Form*. -| As already mentioned above, groups can also be created from the Django-based *Admin Interface* of GeoNode. -| The *Groups* link of the *AUTHENTICATION AND AUTHORIZATION* section allows to manage basic Django groups which only care about permissions. -| To create a GeoNode group you should take a look at the *GROUPS* section. +As already mentioned above, groups can also be created from the Django-based *Admin Interface* of GeoNode. +The *Groups* link of the *AUTHENTICATION AND AUTHORIZATION* section allows to manage basic Django groups which only care about permissions. +To create a GeoNode group you should take a look at the *GROUPS* section. ![groups_admin_section](img/groups_admin_section.png) @@ -143,8 +143,8 @@ On the *Group Profile Form* page you can insert a logo from your disk by click o ![editing_group_logo](img/editing_group_logo.png) -| Click on `Update` to apply the changes. -| Take a look at your group now, you should be able to see that logo. +Click on `Update` to apply the changes. +Take a look at your group now, you should be able to see that logo. ![group_logo](img/group_logo.png) @@ -156,12 +156,12 @@ The `Manage Group Members` link opens the *Group Members Page* which shows *Grou **Managers** can edit group details, can delete the group, can see the group activities and can manage memberships. Other **Members** can only see the group activities. -| In Public Groups, users can join the group without any approval. - Other types of groups require the user to be invited by the group managers. -| Only group managers can *Add new members*. - In the picture below, you can see the manager can search for users by typing their names into the *User Identifiers* search bar. - Once found, he can add them to the group by clicking the `Add Group Members` button. - The *Assign manager role* flag implies that all the users found will become managers of the group. +In Public Groups, users can join the group without any approval. +Other types of groups require the user to be invited by the group managers. +Only group managers can *Add new members*. +In the picture below, you can see the manager can search for users by typing their names into the *User Identifiers* search bar. +Once found, he can add them to the group by clicking the `Add Group Members` button. +The *Assign manager role* flag implies that all the users found will become managers of the group. ![add_new_member](img/add_new_member.png) @@ -171,4 +171,4 @@ The following picture shows you the results. ![new_members](img/new_members.png) -If you want to change the role of group members after adding them, you can use the "promote" button to make a member into a manager, and the "demote" button to make a manager into a regular member. \ No newline at end of file +If you want to change the role of group members after adding them, you can use the "promote" button to make a member into a manager, and the "demote" button to make a manager into a regular member. diff --git a/docs/src/admin/admin_panel/oauth.md b/docs/src/admin/admin_panel/oauth.md index d5c510afbef..a9250e87024 100644 --- a/docs/src/admin/admin_panel/oauth.md +++ b/docs/src/admin/admin_panel/oauth.md @@ -1,11 +1,11 @@ # OAuth2 Access Tokens This small section won't cover entirely the GeoNode OAuth2 security integration, this is explained in detail in other sections of the documentation -(refer to :ref:`oauth2_fixtures_and_migration` and :ref:`oauth2_tokens_and_sessions`). +(refer to the [Django OAuth Toolkit Admin Setup](../../setup/configuration/components/geonode_security_backend/geonode_security_backend.md#django-oauth-toolkit-admin-setup) and [GeoNode / GeoServer Authentication Mechanism](../../setup/configuration/components/geonode_security_backend/geonode_security_backend.md#geonode-geoserver-authentication-mechanism) sections). -Here we will focus mainly on the :guilabel:`Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel items with a specific attention to the ``Access tokens`` management. +Here we will focus mainly on the `Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel items with a specific attention to the ``Access tokens`` management. -The :guilabel:`Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel (as shown in the figure below) allows an admin to manage everything related to +The `Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel (as shown in the figure below) allows an admin to manage everything related to GeoNode OAuth2 grants and permissions. As better explained in other sections of the documentation, this is needed to correctly handle the communication between GeoNode and GeoServer. @@ -14,24 +14,24 @@ As better explained in other sections of the documentation, this is needed to co Specifically from this panel an admin can create, delete or extend OAuth2 ``Access tokens``. -The section :ref:`oauth2_tokens_and_sessions` better explains the concepts behind OAuth2 sessions; we want just to refresh the mind here +The [GeoNode / GeoServer Authentication Mechanism](../../setup/configuration/components/geonode_security_backend/geonode_security_backend.md#geonode-geoserver-authentication-mechanism) section better explains the concepts behind OAuth2 sessions; we want just to refresh the mind here about the basic concepts: -* If the `SESSION_EXPIRED_CONTROL_ENABLED <../../basic/settings/index.html#session-expired-control-enabled>`_ setting is set to `True` (by default it is set to `True`) +* If the `SESSION_EXPIRED_CONTROL_ENABLED` setting is set to `True` (by default it is set to `True`) a registered user cannot login to neither GeoNode nor GeoServer without a valid ``Access token``. * When logging-in into GeoNode through the sign-up form, GeoNode checks if a valid ``Access token`` exists and it creates a new one if not, or extends the existing one if expired. -* New ``Access tokens`` expire automatically after `ACCESS_TOKEN_EXPIRE_SECONDS <../../basic/settings/index.html#access-token-expire-seconds>`_ setting (by default 86400) +* New ``Access tokens`` expire automatically after the [`ACCESS_TOKEN_EXPIRE_SECONDS`](../../setup/configuration/settings.md) setting (by default 86400) * When an ``Access token`` expires, the user will be kicked out from the session and forced to login again ## Create a new token or extend an existing one -It is possible from the :guilabel:`Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel to create a new ``Access token`` for a user. +It is possible from the `Admin > DJANGO/GEONODE OAUTH TOOLKIT` panel to create a new ``Access token`` for a user. -In order to do that, just click on the :guilabel:`Add` button beside ``Access tokens`` topic +In order to do that, just click on the `Add` button beside ``Access tokens`` topic ![oauth2-tokens/admin-panel-tokens-0001](img/oauth2-tokens/admin-panel-tokens-0002.png) @@ -58,7 +58,7 @@ select the followings: ![oauth2-tokens/admin-panel-tokens-0003b](img/oauth2-tokens/admin-panel-tokens-0003c.png) -5. ``Expires``; select an expiration date by using the :guilabel:`date-time` widgets. +5. ``Expires``; select an expiration date by using the `date-time` widgets. ![oauth2-tokens/admin-panel-tokens-0003b](img/oauth2-tokens/admin-panel-tokens-0003d.png) @@ -67,7 +67,7 @@ select the followings: ![oauth2-tokens/admin-panel-tokens-0003b](img/oauth2-tokens/admin-panel-tokens-0003e.png) -Do not forget to :guilabel:`Save`. +Do not forget to `Save`. From now on, GeoNode will use this ``Access Token`` to control the user session (notice that the user need to login again if closing the browser session), and the user will be able to access the OWS Services by using the new ``Access Token``, e.g.: @@ -87,4 +87,4 @@ force its session to expire. Remember that the user could activate another session by logging-in again on GeoNode with its credentials. -In order to be sure the user won't force GeoNode to refresh the token, reset first its password or de-activate it. \ No newline at end of file +In order to be sure the user won't force GeoNode to refresh the token, reset first its password or de-activate it. diff --git a/docs/src/admin/default_lang.md b/docs/src/admin/default_lang.md deleted file mode 100644 index 28561c0892e..00000000000 --- a/docs/src/admin/default_lang.md +++ /dev/null @@ -1,75 +0,0 @@ -# Changing the Default Language - -GeoNode's default language is English, but GeoNode users can change the interface language with the pulldown menu at the top-right of most GeoNode pages. Once a user selects a language, GeoNode remembers that language for subsequent pages. - -## GeoNode Configuration - -As root, edit the GeoNode config file `/home/geonode/geonode/geonode/settings.py`, or `/etc/geonode/settings.py` if GeoNode has been installed using **apt-get**, and change `LANGUAGE_CODE` to the desired default language. - -!!! Note - A list of language codes can be found in the global Django config file `/usr/local/lib/python2.7/dist-packages/django/conf/global_settings.py`, or `/var/lib/geonode/lib/python2.7/site-packages/django/conf/global_settings.py` if GeoNode has been installed using **apt-get**. - -For example, to make French the default language use: - -```python -LANGUAGE_CODE = 'fr' -``` - -Unfortunately, Django overrides this setting, giving the language setting of a user's browser priority. For example, if `LANGUAGE_CODE` is set to French, but the user has configured their operating system for Spanish, they may see the Spanish version when they first visit GeoNode. - -## Additional Steps - -If this is not the desired behavior, and all users should initially see the default `LANGUAGE_CODE`, regardless of their browser settings, do the following steps to ensure Django ignores the browser language settings. Users can always use the pulldown language menu to change the language at any time. - -As **root**, create a new directory within GeoNode's site packages: - -```bash -mkdir /usr/lib/python2.7/dist-packages/setmydefaultlanguage -``` - -or: - -```bash -mkdir /var/lib/geonode/lib/python2.7/site-packages/setmydefaultlanguage -``` - -if GeoNode has been installed using **apt-get**. - -As root, create and edit a new file `/usr/lib/python2.7/dist-packages/setmydefaultlanguage/__init__.py` and add the following lines: - -```python -class ForceDefaultLanguageMiddleware(object): - """ - Ignore Accept-Language HTTP headers - - This will force the I18N machinery to always choose settings.LANGUAGE_CODE - as the default initial language, unless another one is set via sessions or cookies - - Should be installed *before* any middleware that checks request.META['HTTP_ACCEPT_LANGUAGE'], - namely django.middleware.locale.LocaleMiddleware - """ - def process_request(self, request): - if request.META.has_key('HTTP_ACCEPT_LANGUAGE'): - del request.META['HTTP_ACCEPT_LANGUAGE'] -``` - -At the end of the GeoNode configuration file `/home/geonode/geonode/geonode/settings.py`, or `/etc/geonode/settings.py` if GeoNode has been installed using **apt-get**, add the following lines to ensure the above class is executed: - -```python -MIDDLEWARE_CLASSES += ( - 'setmydefaultlanguage.ForceDefaultLanguageMiddleware', -) -``` - -## Restart - -You need to restart GeoNode according to the installation method you have chosen. - -For example, if you are using `NGINX` with `UWSGI`, as root you need to run the following commands: - -```bash -service uwsgi restart -service nginx restart -``` - -Please refer to Translating GeoNode for information on editing GeoNode pages in different languages and creating new GeoNode translations. diff --git a/docs/src/admin/management_commands/gwc.md b/docs/src/admin/management_commands/gwc.md new file mode 100644 index 00000000000..d64d54cb860 --- /dev/null +++ b/docs/src/admin/management_commands/gwc.md @@ -0,0 +1,52 @@ +# Handle GWC tile layers + +GeoWebCache (GWC) is a tile caching mechanism that can be used to speed up the rendering of map layers. + +The `gwc` management command allows you to manage GWC tile layers for your GeoNode instance. + +## Create GWC Tile Layers + +To create GWC tile layers for all your GeoNode layers configured within the local GeoServer, +run the following command: + +```bash +DJANGO_SETTINGS_MODULE=geonode.settings python manage.py gwc create --all +``` + +This will create GWC tile layers for all your GeoNode layers. + +In case you want to create GWC tile layers for a specific layer, you can use the `--layer` option followed by the layer name: + +```bash +DJANGO_SETTINGS_MODULE=geonode.settings python manage.py gwc create --layer +``` + +You can specify multiple layers by repeating the `--layer` option: + +```bash +DJANGO_SETTINGS_MODULE=geonode.settings python manage.py gwc create --layer --layer +``` + +The `gwc create` command is usually quite conservative and will not mess with GWC tile layers if they already exist. +However, it also accepts the `--force` option, which forces the creation of GWC tile layers even if they already exist. +This may be useful if you want to recreate GWC tile layers for some reason, for example reset GWC tile layers after a GeoServer layer update. + +Notice: the `gwc create` command replaces the old `create_tile_layers` command, which is now deprecated and will be removed in a future release. + +## Truncate GWC Tile Layers + +To truncate GWC tile layers for all your GeoNode layers, run the following command: + +```bash +DJANGO_SETTINGS_MODULE=geonode.settings python manage.py gwc truncate --all +``` + +This will truncate GWC tile layers for all your GeoNode layers. + +In case you want to truncate GWC tile layers for one or more specific layers, +you can use the `--layer` option followed by the layer name: + +```bash +DJANGO_SETTINGS_MODULE=geonode.settings python manage.py gwc truncate --layer [--layer ...] +``` + diff --git a/docs/src/admin/thesauri/thesauri.md b/docs/src/admin/thesauri/thesauri.md index 59f0577105d..74f3996deee 100644 --- a/docs/src/admin/thesauri/thesauri.md +++ b/docs/src/admin/thesauri/thesauri.md @@ -92,6 +92,7 @@ GeoNode provides a single command (``thesaurus``) with multiple actions: * ``list``: list existing thesauri * ``load``: load a RDF file * ``dump``: dump a thesaurus into a file +* ``autoload``: automatically discover and load all thesauri shipped by installed apps .. code-block:: @@ -102,12 +103,13 @@ GeoNode provides a single command (``thesaurus``) with multiple actions: [--format {json-ld,n3,nt,pretty-xml,sorted-xml,trig,ttl,xml}] [--default-lang LANG] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] [--skip-checks] - [{list,load,dump}] + [{list,load,dump,autoload}] - Handles thesaurus commands ['list', 'load', 'dump'] + Handles thesaurus commands ['list', 'load', 'dump', 'autoload'] positional arguments: - {list,load,dump} thesaurus operation to run + {list,load,dump,autoload} + thesaurus operation to run options: -h, --help show this help message and exit @@ -227,6 +229,63 @@ In order to only export the entries we edited, we'll issue the command:: python manage.py thesaurus dump -i labels-i18n --include "proj1_*" --include "*_ovr" -f labels-i18n.proj1.rdf +### Auto-loading thesauri: ``thesaurus autoload`` + +The ``autoload`` subcommand scans every installed Django app for a ``thesauri/`` directory +at the top level of the app package, then loads all ``.rdf`` files it finds there. +This is how GeoNode and third-party apps can ship thesauri that are loaded automatically at start-up. + +```bash +python manage.py thesaurus autoload +``` + +For each ``.rdf`` file discovered, the command runs the equivalent of ``thesaurus load --action update``, +so the operation is **idempotent**: running it multiple times will not create duplicates; instead, +existing records are updated and missing ones are created. + +**Convention for app-provided thesauri** + +Place one or more ``.rdf`` files inside a ``thesauri/`` directory at the root of your app package: + +``` +my_geonode_app/ + thesauri/ + my_vocabulary.rdf + another_vocab.rdf + models.py + ... +``` + +All ``.rdf`` files in that directory are picked up automatically whenever ``thesaurus autoload`` +(or ``invoke loadthesauri``) is executed. + +!!! note + The ``autoload`` command is automatically run during GeoNode's Docker container start-up sequence (see [Initialization at boot](#initialization-at-boot)). + + +## Initialization at boot { #initialization-at-boot } + +When GeoNode starts (e.g. via the Docker entrypoint), the following initialization steps are executed in order: + +1. **Database migrations** – applies any pending schema migrations. +2. **Fixtures** – loads default OAuth2 apps, admin user, and site data (only on first boot or when ``FORCE_REINIT=true``). +3. **Static files** – collects static assets. +4. **Thesauri autoload** – runs ``thesaurus autoload`` to load or update all ``.rdf`` files found in any installed app's ``thesauri/`` directory. This step runs on **every** boot so that thesaurus updates shipped with an upgraded app are applied automatically. + +To run the thesaurus autoload step manually: + +```bash +# Inside the GeoNode container +python manage.py thesaurus autoload +``` + +Or using the invoke task: + +```bash +invoke loadthesauri +``` + + ## Configuring a Thesaurus diff --git a/docs/src/overview/basics.md b/docs/src/overview/basics.md index f516aa3abb4..fa794a5fd4c 100644 --- a/docs/src/overview/basics.md +++ b/docs/src/overview/basics.md @@ -1,20 +1,9 @@ # GeoNode Basics { #geonode_basics } -![](img/geonode.png){ align=center } - -is a platform for the management and publication of geospatial data. +GeoNode is a platform for the management and publication of geospatial data. It brings together mature open-source software projects under an easy to use interface. -![](img/gn_simplified_architecture.png){ align=center } -/// caption -*GeoNode simplified architecture* -/// - -## *With GeoNode, non-specialized users can share data and create interactive maps.* - -![](img/gn_is_made_for.png){ align=center } -![](img/gn_publication_data.png){ align=center } -![](img/gn_publication_data_2.png){ align=center } +*With GeoNode, non-specialized users can share data and create interactive maps.* ## Geospatial data storage diff --git a/docs/src/overview/img/geonode.png b/docs/src/overview/img/geonode.png deleted file mode 100644 index 2419c3f26b1..00000000000 Binary files a/docs/src/overview/img/geonode.png and /dev/null differ diff --git a/docs/src/overview/img/gn_is_made_for.png b/docs/src/overview/img/gn_is_made_for.png deleted file mode 100644 index 1b09eabb7fc..00000000000 Binary files a/docs/src/overview/img/gn_is_made_for.png and /dev/null differ diff --git a/docs/src/overview/img/gn_publication_data.png b/docs/src/overview/img/gn_publication_data.png deleted file mode 100644 index a01cdb85121..00000000000 Binary files a/docs/src/overview/img/gn_publication_data.png and /dev/null differ diff --git a/docs/src/overview/img/gn_publication_data_2.png b/docs/src/overview/img/gn_publication_data_2.png deleted file mode 100644 index f74438eb966..00000000000 Binary files a/docs/src/overview/img/gn_publication_data_2.png and /dev/null differ diff --git a/docs/src/overview/img/gn_simplified_architecture.png b/docs/src/overview/img/gn_simplified_architecture.png deleted file mode 100644 index a530d4064c9..00000000000 Binary files a/docs/src/overview/img/gn_simplified_architecture.png and /dev/null differ diff --git a/docs/src/setup/bare/project-bare-installation.md b/docs/src/setup/bare/project-bare-installation.md index 68d3d517e0c..97dedb309fc 100644 --- a/docs/src/setup/bare/project-bare-installation.md +++ b/docs/src/setup/bare/project-bare-installation.md @@ -38,27 +38,8 @@ This will clone the ``master`` branch. You will have to checkout the desidered b As an example, if you want to generate a project for GeoNode 4.4.3 run the following: ```bash -git checkout -b 4.4.3 -``` - -### Generate a custom GeoNode project - -This is the most important part for the GeoNode project installation. Before building the project, you have to generate custom GeoNode project intance, using the `GeoNode Template` - -!!! Note - We will call our instance my_geonode. You can change the name at your convenience. - -```bash -# Create and activate a Python environment called my_geonode_env -mkdir path/to/.venvs -python3 -m venv /path/to/.venvs/my_geonode_env -source /path/to/.venvs/my_geonode_env/bin/activate - -# Install Django in the activated Python environment -pip install Django==5.2.8 - -cd /opt/geonode_projects -django-admin startproject --template=./geonode-project -e py,sh,md,rst,json,yml,ini,env,sample,properties -n monitoring-cron -n Dockerfile my_geonode +cd geonode-project +git checkout -b 5.1.0 ``` ### Install the Python requrements diff --git a/docs/src/setup/bare/vanilla-bare-installation.md b/docs/src/setup/bare/vanilla-bare-installation.md index 8ce32849d1f..452f750d9bd 100644 --- a/docs/src/setup/bare/vanilla-bare-installation.md +++ b/docs/src/setup/bare/vanilla-bare-installation.md @@ -155,16 +155,16 @@ local all postgres trust # TYPE DATABASE USER ADDRESS METHOD # "local" is for Unix domain socket connections only -local all all md5 +local all all scram-sha-256 # IPv4 local connections: -host all all 127.0.0.1/32 md5 +host all all 127.0.0.1/32 scram-sha-256 # IPv6 local connections: -host all all ::1/128 md5 +host all all ::1/128 scram-sha-256 # Allow replication connections from localhost, by a user with the # replication privilege. local replication all peer -host replication all 127.0.0.1/32 md5 -host replication all ::1/128 md5 +host replication all 127.0.0.1/32 scram-sha-256 +host replication all ::1/128 scram-sha-256 ``` !!! Warning @@ -195,7 +195,9 @@ psql -U geonode geonode_data After the creation of the databases, you need to apply database migrations: ```bash -cd /opt/geonode_projects/my_project +cd /opt/geonode +# Load the env file. Here we use .env_dev but you can replace it with yours +set -a && source .env_dev && set +a # Run migrations for the my_geonode database python manage.py migrate # Run migrations for the my_geonode_data database @@ -493,7 +495,7 @@ sudo vim /opt/data/geoserver_data/geofence/geofence-datasource-ovr.properties And paste the following code by replace the placehoders with the required files ```bash -ggeofenceVendorAdapter.databasePlatform=org.hibernate.spatial.dialect.postgis.PostgisDialect +geofenceVendorAdapter.databasePlatform=org.hibernate.spatial.dialect.postgis.PostgisDialect geofenceDataSource.driverClassName=org.postgresql.Driver geofenceDataSource.url=jdbc:postgresql://localhost:5432/geonode_data geofenceDataSource.username=geonode diff --git a/docs/src/setup/configuration/settings.md b/docs/src/setup/configuration/settings.md index a2ec0ae7e32..b5935ffca09 100644 --- a/docs/src/setup/configuration/settings.md +++ b/docs/src/setup/configuration/settings.md @@ -1391,6 +1391,25 @@ If ``RESOURCE_OWNERSHIP_ADMIN_USERNAME`` is not set, GeoNode defaults to ``admin ## S +**SAFE_URL_TRUSTED_HOSTS** + + +: - Default: ``[]`` + - Env: ``SAFE_URL_TRUSTED_HOSTS`` + +A list of trusted hosts that bypass GeoNode's URL safety validation. The most common use case is allowing connections to remote services hosted on private networks (e.g. corporate intranet, VPN). Hosts resolving to loopback, link-local, multicast or reserved IP addresses, as well as hosts not resolvable via public DNS, are also blocked by default and can be allowed through this setting. + +Hosts must be specified in ``domain:port`` format. Standard ports (``80`` for HTTP, ``443`` for HTTPS) are inferred automatically from the URL scheme, so they must still be listed explicitly in this setting. + +Example: + +```python +SAFE_URL_TRUSTED_HOSTS = ['internal.geoserver.example.com:8443', 'internal.geoserver.example.com:443'] +``` + +!!! warning + Only add hosts you fully trust. + **SEARCH_FILTERS** @@ -1637,3 +1656,10 @@ class MyObject(): : Default: ``'ALLOW-FROM %s' % SITEURL`` This is a [Django setting](https://docs.djangoproject.com/en/3.2/ref/clickjacking/#setting-x-frame-options-for-all-responses) + + +**XLSX_UPLOAD_ENABLED** + +: Default: `False` + + Enable or not the upload of XLSX / XLS files \ No newline at end of file diff --git a/docs/src/setup/docker/project-docker-installation.md b/docs/src/setup/docker/project-docker-installation.md index 8eca18ccf00..d94909f4622 100644 --- a/docs/src/setup/docker/project-docker-installation.md +++ b/docs/src/setup/docker/project-docker-installation.md @@ -10,36 +10,19 @@ cd ~/geonode_projects git clone https://github.com/GeoNode/geonode-project.git ``` -This will clone the `master` branch. You will have to checkout the desidered branch or tag. As an example, if you want to generate a propject for GeoNode 4.4.3 you will docker. +This will clone the `master` branch. You will have to checkout the desidered branch or tag. As an example, if you want to generate a propject for GeoNode 5.1.0 run the following: ```bash cd geonode-project -git checkout -b 4.4.3 +git checkout -b 5.1.0 ``` -### Generate a custom GeoNode project - -This is the most important part for the GeoNode project installation. Before building the project, you have to generate custom GeoNode project intance, using the `GeoNode Template` - !!! Note - We will call our instance my_geonode. You can change the name at your convenience. - -```bash -# Create and activate a Python environment called my_geonode_env -mkdir path/to/.venvs -python3 -m venv /path/to/.venvs/my_geonode_env -source /path/to/.venvs/my_geonode_env/bin/activate - -# Install Django in the activated Python environment -pip install Django==5.2.8 - -cd ~/geonode_projects -django-admin startproject --template=./geonode-project -e py,sh,md,rst,json,yml,ini,env,sample,properties -n monitoring-cron -n Dockerfile my_geonode -``` + You can replace the release number `5.1.0` with the latest one. You can find the releases [here](https://github.com/GeoNode/geonode-project/releases/) ### Prepare the .env file -Navigate to `my_geonode` folder and create the .env file by using the `create-envfile` script: +Go inside the `geonode-project` folder and create the .env file by using the `create-envfile` script: ```bash cd my_geonode diff --git a/docs/src/setup/docker/vanilla-docker-installation.md b/docs/src/setup/docker/vanilla-docker-installation.md index 5136c1a2d0d..bef557550a8 100644 --- a/docs/src/setup/docker/vanilla-docker-installation.md +++ b/docs/src/setup/docker/vanilla-docker-installation.md @@ -81,6 +81,15 @@ Executing UWSGI server uwsgi --ini /usr/src/app/uwsgi.ini for Production [uWSGI] getting INI configuration from /usr/src/app/uwsgi.ini ``` +The container performs these initialization steps before starting the application server: + +1. **Database migrations** – applies any pending schema migrations. +2. **Fixtures** – loads default OAuth2 apps, admin user and site data (only on first boot or when ``FORCE_REINIT=true``). +3. **Static files** – collects static assets. +4. **Thesauri autoload** – scans all installed apps for a ``thesauri/`` directory and loads (or updates) any ``.rdf`` files found there. This makes sure thesauri shipped by GeoNode apps are always up-to-date. + +See [Thesauri – Initialization at boot](../../../admin/thesauri/thesauri.md#initialization-at-boot) for more details on the thesaurus autoload step. + To exit just hit `CTRL+C`. This message means that the GeoNode containers have been started. Browsing to `http://localhost/` will show the GeoNode home page. You should be able to successfully log with the credentials of admin user which are defined in the .env file and start using it right away. diff --git a/docs/src/setup/prod_enhancements/prod_enhancements.md b/docs/src/setup/prod_enhancements/prod_enhancements.md index 122b037b0fc..673a926c272 100644 --- a/docs/src/setup/prod_enhancements/prod_enhancements.md +++ b/docs/src/setup/prod_enhancements/prod_enhancements.md @@ -93,13 +93,13 @@ docker-compose restart geoserver - **WMS**: `Raster Rendering Options` allows you to tune up the WMS output for better performance or quality. Best Performance: `Nearest Neighbour` - Best Quality: `Bicubic`. - !!! Warning - Raster images should always be optimized before being ingested into GeoNode. The general recommendation is to **never** upload a non-processed GeoTIFF image to GeoNode. + !!! warning + Raster images should always be optimized before being ingested into GeoNode. The general recommendation is to **never** upload a non-processed GeoTIFF image to GeoNode. - Further details: + Further details: - - [Enterprise raster training](https://geoserver.geo-solutions.it/edu/en/enterprise/raster.html) - - [Advanced GDAL raster data](https://geoserver.geo-solutions.it/edu/en/raster_data/advanced_gdal/index.html) + - [Enterprise raster training](https://geoserver.geo-solutions.it/edu/en/enterprise/raster.html) + - [Advanced GDAL raster data](https://geoserver.geo-solutions.it/edu/en/raster_data/advanced_gdal/index.html) ![](img/production_geoserver_006.png){ align=center width="350px" } /// caption @@ -188,7 +188,7 @@ mv postgis-jdbc-1.3.3.jar /usr/local/tomcat/webapps/geoserver/WEB-INF/lib/ The container is ready to be restarted now. -!!! Warning +!!! warning Remember to do a **soft restart** otherwise the `WEB-INF/lib` JARs will be reset to the original state. ```bash diff --git a/docs/src/user-guide/datasets/remote.md b/docs/src/user-guide/datasets/remote.md index 6787e607f4a..4b6abbba08a 100644 --- a/docs/src/user-guide/datasets/remote.md +++ b/docs/src/user-guide/datasets/remote.md @@ -29,6 +29,9 @@ Once the service has been configured, you can select the resources you are inter From the page where the services are listed, it is possible to click on the Title of a service, which will open the Service Details page. If you want to import more resources from that service, you can click on the **Import Service Resources** button. +!!! note + To connect to a remote service hosted on a private network, see [`SAFE_URL_TRUSTED_HOSTS`](../../setup/configuration/settings.md#safe_url_trusted_hosts). + ### Remote 3D Tiles The GeoNode client supports visualization of [3D Tiles](https://docs.mapstore.geosolutionsgroup.com/en/latest/user-guide/catalog/#3d-tiles-catalog) thanks to the capabilities fo the MapStore framework on which it is based. 3D Tiles tilesets can be published either from a file upload (.zip file containing the tileset) or by reference of a remotwly published tileset, served over HTTP(S). diff --git a/docs/src/user-guide/default-language.md b/docs/src/user-guide/default-language.md new file mode 100644 index 00000000000..3f49b4da01b --- /dev/null +++ b/docs/src/user-guide/default-language.md @@ -0,0 +1,62 @@ +# Default Language Management + +GeoNode supports dynamic language selection through both the user interface and the user profile settings. + +## Authenticated Users + +Authenticated users can change the application language in two different ways: + +- Through the **topbar language switcher** +- Through the **Language field** in their User Profile + +![](img/topbar_language_switcher.png){ align=center } +/// caption +Topbar language switcher. +/// + +![](img/profile_lang_switcher.png){ align=center } +/// caption +Language field in the user profile. +/// + +Both mechanisms are fully synchronized. + +When a user changes the language using the topbar switcher, the selected language is automatically persisted in the user profile and stored in the database. Similarly, updating the language from the User Profile immediately affects the language used by the interface. + +This ensures that: + +- the selected language is persistent across sessions and browser restarts +- the same language preference is consistently applied after login +- the topbar switcher and the profile language always remain synchronized + +The stored language preference is automatically applied for authenticated users through middleware during each request. + +--- + +## Anonymous Users + +Anonymous users can also change the language using the topbar language switcher. + +Since anonymous users do not have a profile stored in the database, the selected language is stored only in the browser through Django's language cookie mechanism. + +As a result: + +- the language preference affects only the current browser +- the preference is not persisted in the GeoNode database +- different browsers or devices may use different languages independently + +--- + +## Read-Only Mode + +When the GeoNode instance is configured in **read-only mode**, persistent database updates are disabled. + +In this scenario: + +- authenticated users can still change the interface language using the topbar switcher +- the selected language is applied only for the current browser/session +- the language preference is not written to the user profile in the database + +This behavior ensures that language switching remains available even while the platform prevents content modifications. + +The read-only restriction therefore affects only the persistence of the language preference, not the ability to temporarily change the interface language. diff --git a/docs/src/user-guide/img/profile_lang_switcher.png b/docs/src/user-guide/img/profile_lang_switcher.png new file mode 100644 index 00000000000..2e9ee653fe2 Binary files /dev/null and b/docs/src/user-guide/img/profile_lang_switcher.png differ diff --git a/docs/src/user-guide/img/topbar_language_switcher.png b/docs/src/user-guide/img/topbar_language_switcher.png new file mode 100644 index 00000000000..e0be594b486 Binary files /dev/null and b/docs/src/user-guide/img/topbar_language_switcher.png differ diff --git a/docs/src/user-guide/resource_info.md b/docs/src/user-guide/resource_info.md index 64367cb50bf..437173a18ee 100644 --- a/docs/src/user-guide/resource_info.md +++ b/docs/src/user-guide/resource_info.md @@ -71,6 +71,27 @@ From the lower right toolbar on the thumbnail part of the properties panel, it i - Copy the resource URL - Copy the OGC resource web services URL (in the case of a `Dataset`) +## Cloning a resource + +Cloning creates a new, fully independent resource. It gets its own UUID, its own permissions record, and, for a `Dataset`, its own copy of the data on the GIS backend. Ownership of the clone is transferred to whoever triggers it, regardless of who owned the source. + +What is carried over from the source: + +- Metadata: title, abstract, category, license, and every other descriptive field +- Keywords, regions, and thesaurus keywords +- Contacts and their roles (point of contact, metadata author, and so on) +- Geographic access limits (per-user and per-group) +- Permissions: the clone starts with the same permission spec as the source, not the default permissions a newly created resource would get +- Linked resources (e.g. a `Map`'s linked `Datasets`) +- Type-specific data: a `Dataset`'s attribute table, a `Map`'s layers, and the underlying files/assets + +What does not carry over: + +- The owner, which becomes the user who triggered the clone +- The `featured` flag, always reset to off on the clone + +Because the clone owns its own copy of everything above rather than sharing rows with the source, deleting the source resource afterward does not affect the clone. + You can access the resource details page by clicking the button on the right (`View dataset` in the case of a `dataset`) in the overview panel. That page looks like the one shown in the picture below. diff --git a/entrypoint.sh b/entrypoint.sh index ed9469efac2..18c8ab5269f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -56,6 +56,7 @@ else fi invoke statics + invoke loadthesauri echo "Executing UWSGI server $cmd for Production" fi diff --git a/geonode/__init__.py b/geonode/__init__.py index a5ad6613ff7..e18c1b7350c 100644 --- a/geonode/__init__.py +++ b/geonode/__init__.py @@ -28,6 +28,12 @@ def get_version(): return geonode.version.get_version(__version__) +# PEP 440 compliant version string, referenced by pyproject.toml's dynamic +# version (a plain string attribute, so setuptools doesn't have to stringify the +# __version__ tuple, which would produce an invalid version like "5.1.0.final.0"). +__version_str__ = get_version() + + def main(_, **settings): from django.core.wsgi import get_wsgi_application diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index cf98190ac8e..9ab377cab32 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -17,6 +17,7 @@ # ######################################################################### from geonode.base.enumerations import LAYER_TYPES +import json import logging from django.db.models import Q @@ -45,6 +46,7 @@ from geonode.groups.models import GroupProfile from geonode.utils import check_ogc_backend from geonode.security.utils import get_visible_resources +from geonode.metadata.models import SparseField from .authentication import OAuthAuthentication from .authorization import GeoNodeAuthorization, GeonodeApiKeyAuthentication @@ -133,6 +135,11 @@ class CommonModelApi(ModelResource): "metadata_only", ] + @staticmethod + def _extract_deprecated_metadata_filters(filters): + """Extract legacy ``metadata__*`` query params for sparse-field lookup.""" + return {key: value for key, value in filters.items() if key.startswith("metadata__")} + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): if filters is None: filters = {} @@ -142,7 +149,9 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): if "app_type__in" in filters: orm_filters.update({"resource_type": filters["app_type__in"].lower()}) - _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith("metadata__")} + # Deprecated compatibility: keep supporting metadata__* filters + # by mapping them to SparseField lookups in apply_filters. + _metadata = self._extract_deprecated_metadata_filters(filters) if _metadata: orm_filters.update({"metadata_filters": _metadata}) @@ -208,7 +217,7 @@ def apply_filters(self, request, applicable_filters): filtered = self.filter_h_keywords(filtered, keywords) if metadata_filters: - filtered = filtered.filter(**metadata_filters) + filtered = self.filter_sparse_fields(filtered, metadata_filters) # return filtered return get_visible_resources( @@ -236,6 +245,75 @@ def filter_h_keywords(self, queryset, keywords): filtered = queryset return filtered + def filter_sparse_fields(self, queryset, metadata_filters): + """ + Filter queryset by sparse field values (metadata custom fields). + + Queryset is filtered by interrogating SparseField entries that match + the given metadata filter specifications. + + Args: + queryset: ResourceBase queryset to filter + metadata_filters: dict with keys like "metadata__key" and values to match + + Returns: + Filtered queryset containing only resources with matching sparse fields + """ + if not metadata_filters: + return queryset + + filtered_pks = set() + found_metadata_filter = False + + for filter_key, filter_value in metadata_filters.items(): + # Extract field name from "metadata__fieldname" + if not filter_key.startswith("metadata__"): + continue + + found_metadata_filter = True + field_name = filter_key[len("metadata__") :] + + # Text prefilter to reduce the set to deserialize; the actual + # semantic match is performed after json.loads. + sparse_fields = ( + SparseField.objects.filter(name__startswith="extra_") + .filter(value__icontains=field_name) + .filter(value__icontains=str(filter_value)) + ) + + batch_pks = set() + for sf in sparse_fields: + try: + # ExtraMetadata was a JSONfield + stored_value = json.loads(sf.value) if sf.value and sf.value.startswith("{") else sf.value + except (json.JSONDecodeError, TypeError): + logger.warning( + f"Bad migrated ExtraMetadata into SparseField: {sf.name} for resource {sf.resource.id}:{sf.resource.title}" + ) + continue + + if not isinstance(stored_value, dict): + logger.warning( + f"Unexpected non-dict value in SparseField: {sf.name} for resource {sf.resource.id}:{sf.resource.title}" + ) + continue + + # Compare values in a type-agnostic way. + if str(stored_value.get(field_name, None)) == str(filter_value): + batch_pks.add(sf.resource.pk) + + filtered_pks.update(batch_pks) + + # Filter by the collected PKs + # If we processed metadata filters, return only resources with matching values + if found_metadata_filter: + filtered = queryset.filter(pk__in=filtered_pks) if filtered_pks else queryset.none() + else: + # No metadata filters found, return queryset as-is + filtered = queryset + + return filtered + def get_list(self, request, **kwargs): """ Returns a serialized list of resources. @@ -249,6 +327,7 @@ def get_list(self, request, **kwargs): # impossible. base_bundle = self.build_bundle(request=request) objects = self.obj_get_list(bundle=base_bundle, **self.remove_api_resource_names(kwargs)) + sorted_objects = self.apply_sorting(objects, options=request.GET) paginator = self._meta.paginator_class( diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 3538dc8b987..448c6775bfb 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -37,7 +37,6 @@ from geonode.layers.models import Dataset from geonode.documents.models import Document from geonode.base.models import ( - ExtraMetadata, Thesaurus, ThesaurusLabel, ThesaurusKeyword, @@ -52,6 +51,7 @@ from geonode.tests.base import GeoNodeBaseTestSupport from geonode.base.populate_test_data import all_public, create_models, remove_models from geonode.security.registry import permissions_registry +from geonode.metadata.models import SparseField from geonode.assets.models import Asset from django.core.files.uploadedfile import SimpleUploadedFile from geonode.base.models import Link @@ -536,35 +536,49 @@ def test_category_filters(self): self.assertEqual(len(self.deserialize(resp)["objects"]), 5) def test_metadata_filters(self): - """Test category filtering""" + """Test metadata filtering against sparse fields.""" _r = Dataset.objects.first() - _m = ExtraMetadata.objects.create( + + # Create sparse field using migrated ExtraMetadata format. + SparseField.objects.update_or_create( resource=_r, - metadata={ - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "category", - }, + name="extra_1", + defaults={"value": json.dumps({"category": "category"})}, ) + # Verify sparse field was created + _sf_check = SparseField.objects.filter(resource=_r, name="extra_1") + self.assertTrue(_sf_check.exists(), "SparseField should have been created") + self.assertEqual(_sf_check.first().value, json.dumps({"category": "category"})) + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) - _r.metadata.add(_m) - # check we get the correct layers number returnered filtering on one - # and then two different categories - filter_url = f"{list_url}?metadata__category=category" + # Test 1: Filter for existing category value + filter_url = f"{list_url}?metadata__category=category" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)["objects"]), 1) - filter_url = f"{list_url}?metadata__category=not-existing-category" + result = self.deserialize(resp) + result_count = len(result["objects"]) + logger.debug(f"Test 1 - Filter metadata__category=category: Got {result_count} results (expected 1)") + logger.debug(f"Result PKs: {[obj['id'] for obj in result['objects']]}") + logger.debug(f"Created resource PK: {_r.pk}") + self.assertEqual(result_count, 1, f"Expected 1 result, got {result_count}") + self.assertEqual(result["objects"][0]["id"], _r.pk) + + # Test 2: Filter for non-existing category value + filter_url = f"{list_url}?metadata__category=not-existing-category" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + result = self.deserialize(resp) + result_count = len(result["objects"]) + logger.debug( + f"Test 2 - Filter metadata__category=not-existing-category: Got {result_count} results (expected 0)" + ) + + self.assertEqual(result_count, 0, f"Expected 0 results, got {result_count}") def test_tag_filters(self): """Test keywords filtering""" diff --git a/geonode/assets/local.py b/geonode/assets/local.py index 650fd0cc2dc..d8e62deb3ce 100644 --- a/geonode/assets/local.py +++ b/geonode/assets/local.py @@ -22,6 +22,9 @@ concrete_storage_manager=FileSystemStorageManager(location=os.path.dirname(settings.ASSETS_ROOT)) ) +# extensions served as an attachment (download) instead of rendered inline +FORCE_DOWNLOAD_EXTENSIONS = {"xml", "sld"} + class DefaultLocalLinkUrlHandler: def get_link_url(self, asset: LocalAsset): @@ -256,7 +259,7 @@ def create_response( return HttpResponse("Asset does not contain any data", status=500) if len(asset.location) > 1: - logger.warning("TODO: Asset contains more than one file. Download needs to be implemented") + logger.warning("TODO: Asset contains more than one file. Only first file will be returned") file0 = asset.location[0] if not path: # use the file definition @@ -266,10 +269,6 @@ def create_response( localfile = file0 else: # a specific file is requested - if "/../" in path: # we may want to improve fraudolent request detection - logger.warning(f"Tentative path traversal for asset {asset.id}") - return HttpResponse(f"File not found for asset {asset.id}", status=400) - if os.path.isfile(file0): dir0 = os.path.dirname(file0) elif os.path.isdir(file0): @@ -280,6 +279,13 @@ def create_response( localfile = os.path.join(dir0, path) logger.debug(f"Requested path {dir0} + {path}") + # check the requested file is within the asset's directory + localfile = os.path.realpath(localfile) + realassetpath = os.path.realpath(dir0) + if os.path.commonpath([realassetpath, localfile]) != realassetpath: + logger.error(f"Tentative path traversal for asset {asset.id} on path [{path}]") + return HttpResponse(f"File not found for asset {asset.id}", status=400) + if os.path.isfile(localfile): filename = os.path.basename(localfile) orig_base, ext = os.path.splitext(filename) @@ -300,9 +306,17 @@ def create_response( ) case False: logger.info(f"Returning file '{localfile}' with name '{outname}'") - return DownloadResponse( - _asset_storage_manager.open(localfile).file, basename=f"{outname}", attachment=False + force_download = ext.lower().lstrip(".") in FORCE_DOWNLOAD_EXTENSIONS + response = DownloadResponse( + _asset_storage_manager.open(localfile).file, + basename=f"{outname}", + attachment=force_download, + ) + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["Content-Security-Policy"] = ( + "default-src 'none'; style-src 'unsafe-inline'; sandbox" ) + return response else: logger.warning(f"Internal file {localfile} not found for asset {asset.id}") return HttpResponse(f"Internal file not found for asset {asset.id}", status=404 if path else 500) diff --git a/geonode/assets/tests.py b/geonode/assets/tests.py index 3ccc6ccff6d..db7c6552e23 100644 --- a/geonode/assets/tests.py +++ b/geonode/assets/tests.py @@ -29,6 +29,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.http import StreamingHttpResponse from django.urls import reverse +from django.test import override_settings from rest_framework.test import APITestCase @@ -40,6 +41,8 @@ from geonode.assets.utils import create_asset, create_asset_and_link, unlink_asset from geonode.base.models import ResourceBase, Link from geonode.security.registry import permissions_registry +from rest_framework import status + logger = logging.getLogger(__name__) @@ -415,6 +418,79 @@ def test_download_with_attachment_for_anonymous(self): if resource: resource.delete() + @override_settings(RESOURCE_PUBLISHING=True) + def test_download_respects_resource_visibility(self): + from geonode.resource.registry import dataset_manager + from geonode.layers.models import Dataset + + owner = get_user_model().objects.get(username="admin") + asset = self._setup_test(owner) + + resource = dataset_manager.create( + None, + resource_type=Dataset, + defaults={ + "owner": owner, + "asset": asset, + "is_published": False, + }, + ) + + resource.set_permissions( + { + "users": {"AnonymousUser": ["view_resourcebase", "download_resourcebase"]}, + "groups": {}, + } + ) + + self.client.logout() + response = self.client.get(reverse("assets-download", kwargs={"pk": asset.pk})) + + self.assertEqual(response.status_code, 401) + + def test_cross_download(self): + + admin, _ = get_user_model().objects.get_or_create(username="admin") + user1, _ = get_user_model().objects.get_or_create(username="user1") + user2, _ = get_user_model().objects.get_or_create(username="user2") + + asset_handler = asset_handler_registry.get_default_handler() + + assets = [] + for user, file in ((user1, ONE_JSON), (user2, TWO_JSON)): + asset1 = asset_handler.create( + title="Test Asset1", + description="Description of test asset", + type="NeverMind", + owner=user1, + files=[file], + clone_files=True, + ) + asset1.save() + self.assertIsInstance(asset1, LocalAsset) + assets.append(asset1) + + # check that user1 can access file1 + self.client.force_login(user1) + + args = {"pk": assets[0].pk, "path": "one.json"} + url = reverse("assets-link", kwargs=args) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + # check that user1 can NOT access file2 using asset1 link/download + a2path = assets[1].location[0] + logger.debug(f"Asset path {a2path}") + a2dir = a2path.split("/")[-2] + logger.debug(f"Asset dir {a2dir}") + forged_path = f"../{a2dir}/two.json" # path will be automatically urlencoded into "%2e%2e%2f{a2dir}%2ftwo.json" + + args = {"pk": assets[0].pk, "path": forged_path} + url = reverse("assets-link", kwargs=args) + logger.debug(f"Reverse link URL is {url}") + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + def _setup_test(self, u, _file=ONE_JSON): asset_handler = asset_handler_registry.get_default_handler() asset = asset_handler.create( @@ -588,3 +664,104 @@ def test_delete_asset_and_link(self): self.assertFalse(Asset.objects.filter(pk=asset_pk).exists()) self.assertFalse(Link.objects.filter(pk=self.link1.pk).exists()) self.assertFalse(os.path.exists(asset_file_path)) + + +class AssetViewSetPermissionsTests(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + self.admin = get_user_model().objects.get(username="admin") + self.user = get_user_model().objects.create_user(username="asset_user", password="password") + self.resource = ResourceBase.objects.create(owner=self.admin, title="Private resource") + self.asset, self.link = create_asset_and_link( + self.resource, + self.admin, + [ONE_JSON], + title="Private asset", + ) + + def test_anonymous_cannot_retrieve_private_linked_asset(self): + response = self.client.get(reverse("assets-detail", kwargs={"pk": self.asset.pk})) + + self.assertIn(response.status_code, [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]) + + def test_user_with_view_resourcebase_can_retrieve_linked_asset(self): + self.resource.set_permissions({"users": {self.user.username: ["view_resourcebase"]}, "groups": {}}) + + self.client.force_login(self.user) + response = self.client.get(reverse("assets-detail", kwargs={"pk": self.asset.pk})) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_user_without_change_resourcebase_cannot_patch_linked_asset(self): + self.resource.set_permissions({"users": {self.user.username: ["view_resourcebase"]}, "groups": {}}) + + self.client.force_login(self.user) + response = self.client.patch( + reverse("assets-detail", kwargs={"pk": self.asset.pk}), + data=json.dumps({"title": "SHOULD-NOT-WORK"}), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.asset.refresh_from_db() + self.assertEqual(self.asset.title, "Private asset") + + def test_user_without_change_resourcebase_cannot_delete_linked_asset(self): + self.resource.set_permissions({"users": {self.user.username: ["view_resourcebase"]}, "groups": {}}) + + self.client.force_login(self.user) + response = self.client.delete(reverse("assets-detail", kwargs={"pk": self.asset.pk})) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertTrue(Asset.objects.filter(pk=self.asset.pk).exists()) + + +class PermissionsRegistryAssetPermTests(GeoNodeBaseTestSupport): + """ + The Asset permission logic lives in the permissions registry now (not in the + DRF permission class), so it's tested directly here. + """ + + def setUp(self): + super().setUp() + self.admin = get_user_model().objects.get(username="admin") + self.owner = get_user_model().objects.create_user(username="asset_owner", password="password") + self.other_user = get_user_model().objects.create_user(username="other_user", password="password") + + def test_superuser_always_allowed(self): + asset, _ = create_asset_and_link( + ResourceBase.objects.create(owner=self.owner, title="r1"), self.owner, [ONE_JSON] + ) + self.assertTrue(permissions_registry.user_has_asset_perm(self.admin, asset, method="GET")) + self.assertTrue(permissions_registry.user_has_asset_perm(self.admin, asset, method="DELETE")) + + def test_asset_without_linked_resource_falls_back_to_owner(self): + asset = LocalAsset.objects.create(title="orphan", owner=self.owner, type="test") + + self.assertTrue(permissions_registry.user_has_asset_perm(self.owner, asset, method="GET")) + self.assertTrue(permissions_registry.user_has_asset_perm(self.owner, asset, method="PATCH")) + self.assertFalse(permissions_registry.user_has_asset_perm(self.other_user, asset, method="GET")) + + def test_write_requires_change_resourcebase_on_every_linked_resource(self): + resource = ResourceBase.objects.create(owner=self.owner, title="r2") + asset, _ = create_asset_and_link(resource, self.owner, [ONE_JSON]) + + # only view perm granted -> read ok, write denied + resource.set_permissions({"users": {self.other_user.username: ["view_resourcebase"]}, "groups": {}}) + self.assertTrue(permissions_registry.user_has_asset_perm(self.other_user, asset, method="GET")) + self.assertFalse(permissions_registry.user_has_asset_perm(self.other_user, asset, method="PATCH")) + + # bump to change perm -> write allowed too + resource.set_permissions( + {"users": {self.other_user.username: ["view_resourcebase", "change_resourcebase"]}, "groups": {}} + ) + self.assertTrue(permissions_registry.user_has_asset_perm(self.other_user, asset, method="PATCH")) + + def test_anonymous_denied_on_write(self): + resource = ResourceBase.objects.create(owner=self.owner, title="r3") + asset, _ = create_asset_and_link(resource, self.owner, [ONE_JSON]) + from guardian.shortcuts import get_anonymous_user + + anonymous = get_anonymous_user() + + self.assertFalse(permissions_registry.user_has_asset_perm(anonymous, asset, method="DELETE")) diff --git a/geonode/assets/utils.py b/geonode/assets/utils.py index 1ecd14c1307..45bfcda1310 100644 --- a/geonode/assets/utils.py +++ b/geonode/assets/utils.py @@ -2,6 +2,7 @@ import os.path from django.http import HttpResponse +from django.conf import settings from django.core.exceptions import PermissionDenied from geonode.security.permissions import DOWNLOAD_PERMISSIONS from geonode.assets.handlers import asset_handler_registry @@ -23,7 +24,13 @@ def get_perms_response(request, asset: Asset): logger.debug("Asset: access allowed by user") return None - visibile_res = get_visible_resources(queryset=ResourceBase.objects.filter(link__asset=asset), user=request.user) + visibile_res = get_visible_resources( + queryset=ResourceBase.objects.filter(link__asset=asset), + user=request.user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) if visibile_res.exists(): # retrieving the resource permissions for the given user diff --git a/geonode/assets/views.py b/geonode/assets/views.py index e7d22bcdc4f..ec5826a919a 100644 --- a/geonode/assets/views.py +++ b/geonode/assets/views.py @@ -36,16 +36,29 @@ DynamicSearchFilter, ) from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.security.registry import permissions_registry +from rest_framework import permissions logger = logging.getLogger(__name__) +class UserHasAssetPerms(permissions.BasePermission): + """ + Thin DRF adapter: the actual decision is delegated to the permissions registry, + which knows how to resolve an Asset's perms through its linked ResourceBase(s). + """ + + def has_object_permission(self, request, view, obj): + return permissions_registry.user_has_asset_perm(request.user, obj, method=request.method) + + class AssetViewSet(DynamicModelViewSet): """ API endpoint that allows Assets to be viewed or edited. """ - permission_classes = [IsAuthenticatedOrReadOnly] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasAssetPerms] + http_method_names = ["get", "put", "patch", "delete"] filter_backends = [ DynamicFilterBackend, DynamicSortingFilter, @@ -63,7 +76,7 @@ def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) user = request.user - is_admin = user.is_superuser if user and user.is_authenticated else False + is_admin = user and user.is_authenticated and user.is_superuser if is_admin: pass @@ -104,5 +117,5 @@ def download(self, request, pk=None, path=None, *args, **kwargs): methods=["get"], ) def link(self, request, pk=None, path=None, *args, **kwargs): - logger.warning(f"REQUESTED ASSET LINK FOR PK:{pk} PATH:{path}") + logger.debug(f"REQUESTED ASSET LINK FOR PK:{pk} PATH:{path}") return self._get_file(request, pk, attachment=False, path=path) diff --git a/geonode/base/api/deprecated_extra_metadata.py b/geonode/base/api/deprecated_extra_metadata.py new file mode 100644 index 00000000000..43ee5a02b6c --- /dev/null +++ b/geonode/base/api/deprecated_extra_metadata.py @@ -0,0 +1,353 @@ +# ######################################################################### +# +# Copyright (C) 2025 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ######################################################################### +""" +Deprecated backward-compatible adapters for the ExtraMetadata API. + +These adapters re-expose the old ``/extra_metadata/`` endpoint and the +``metadata`` serializer field using :class:`SparseField` as the storage +backend. They are marked **deprecated** and should be removed after the +deprecation period. + +To remove these adapters: + 1. Delete this file. + 2. Remove the ``metadata`` field and import from ``serializers.py``. + 3. Remove ``DeprecatedExtraMetadataMixin`` and its import from ``views.py``. + 4. Remove deprecated settings from ``settings.py`` + (``DEFAULT_EXTRA_METADATA_SCHEMA``, ``CUSTOM_METADATA_SCHEMA``, + ``EXTRA_METADATA_SCHEMA``, and the ``from schema import Optional`` import). +""" + +import json +import logging +import uuid +import warnings + +from deprecated import deprecated +from rest_framework.decorators import action +from rest_framework.response import Response + +from dynamic_rest.fields.fields import DynamicComputedField + +from django.db import IntegrityError + +from geonode.base.api.permissions import IsOwnerOrAdmin, UserHasPerms +from geonode.base.models import ResourceBase +from geonode.metadata.models import SparseField + +logger = logging.getLogger(__name__) + +DEPRECATION_VERSION = "5.1.0" +DEPRECATION_REASON = ( + "The extra_metadata API is deprecated and will be removed in a future " + "version. Use the sparse fields API instead." +) +SPARSE_FIELD_PREFIX = "extra_" +# SparseField.value is CharField(max_length=1024); entries exceeding this +# limit cannot be stored and are silently skipped with a log warning. +SPARSE_FIELD_VALUE_MAX_LENGTH = 1024 + +# Query parameter names that should *not* be treated as legacy +# ``metadata__`` filters in the deprecated GET endpoint. +_NON_FILTER_QUERY_PARAMS = { + "api_preset", + "page", + "page_size", + "format", + "include[]", + "exclude[]", + "sort[]", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _sparse_fields_for_resource(resource): + """Return SparseField entries that represent migrated ExtraMetadata.""" + return SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + + +def _sparse_to_legacy(sparse_field): + """Convert a SparseField into the old ExtraMetadata representation. + + Returns ``{"id": , ...metadata_dict}`` so existing consumers see the + same shape they used to get. + """ + try: + metadata = json.loads(sparse_field.value) if sparse_field.value else {} + except (json.JSONDecodeError, TypeError): + metadata = {} + if not isinstance(metadata, dict): + logger.warning(f"Skipping unexpected non-dict metadata in SparseField pk={sparse_field.pk}") + metadata = {} + return {**{"id": sparse_field.pk}, **metadata} + + +def _next_sparse_name(resource): + """Generate the next available ``extra_`` name. + + Falls back to a UUID-based suffix if a name collision occurs (e.g. under + concurrent requests), keeping the ``extra_`` prefix intact. + The numeric part is still preferred for readability. + """ + existing = ( + SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + .order_by("-name") + .values_list("name", flat=True) + ) + max_n = 0 + for name in existing: + suffix = name[len(SPARSE_FIELD_PREFIX) :] + try: + max_n = max(max_n, int(suffix)) + except (ValueError, TypeError): + pass + candidate = f"{SPARSE_FIELD_PREFIX}{max_n + 1}" + # Guard against a race: if the candidate already exists, use a UUID suffix + if SparseField.objects.filter(resource=resource, name=candidate).exists(): + candidate = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + return candidate + + +# --------------------------------------------------------------------------- +# Deprecated serializer field (for ``metadata`` on ResourceBaseSerializer) +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataField(DynamicComputedField): + """Deferred computed field that reconstructs the legacy ``metadata`` + representation from :class:`SparseField` entries. + + .. deprecated:: 5.1.0 + Use the sparse fields API instead. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @deprecated(version=DEPRECATION_VERSION, reason=DEPRECATION_REASON) + def get_attribute(self, instance): + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + try: + qs = _sparse_fields_for_resource(instance) + return [_sparse_to_legacy(sf) for sf in qs] + except Exception as e: + logger.exception(e) + return [] + + +# --------------------------------------------------------------------------- +# Deprecated extra_metadata view action mixin +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataMixin: + """Mixin that adds the deprecated ``extra_metadata`` action back to a + ``ViewSet``. + + Import this mixin and add it to your ViewSet's bases to restore the + old ``/{pk}/extra_metadata/`` endpoint backed by SparseFields. + + .. deprecated:: 4.4.0 + Use the sparse fields API instead. + """ + + @action( + detail=True, + methods=["get", "put", "delete", "post"], + url_path=r"extra_metadata", + url_name="extra-metadata", + permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + ) + def extra_metadata(self, request, pk, *args, **kwargs): + """Deprecated endpoint – delegates to SparseField storage.""" + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + logger.warning(DEPRECATION_REASON) + + resource = ResourceBase.objects.filter(pk=pk).first() + if resource is None: + return Response({"detail": "Not found."}, status=404) + + if request.method == "GET": + return self._extra_metadata_get(request, resource) + elif request.method == "POST": + return self._extra_metadata_post(request, resource) + elif request.method == "PUT": + return self._extra_metadata_put(request, resource) + elif request.method == "DELETE": + return self._extra_metadata_delete(request, resource) + + # -- GET ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_get(request, resource): + qs = _sparse_fields_for_resource(resource) + # Support the old query-param filtering (e.g. ?field_name=value) + for key, value in request.query_params.items(): + if key in _NON_FILTER_QUERY_PARAMS or key.startswith("filter{"): + continue + # Old API used metadata__=value JSONField lookups. + # We approximate this by filtering on the JSON string. + filtered = [] + for sf in qs: + try: + meta = json.loads(sf.value) if sf.value else {} + if not isinstance(meta, dict): + logger.warning(f"Skipping unexpected non-dict metadata in SparseField pk={sf.pk}") + continue + except (json.JSONDecodeError, TypeError): + logger.warning(f"Skipping unparsable json in SparseField pk={sf.pk}") + continue + if str(meta.get(key)) == str(value): + filtered.append(sf) + qs = filtered + break # Old API only used the first filter pair + + if isinstance(qs, list): + return Response([_sparse_to_legacy(sf) for sf in qs]) + return Response([_sparse_to_legacy(sf) for sf in qs.iterator()]) + + # -- POST --------------------------------------------------------------- + + @staticmethod + def _extra_metadata_post(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + cleaned = [] + for meta_dict in data: + if not isinstance(meta_dict, dict): + logger.warning(f"Skipping non-dict metadata entry in POST payload: {meta_dict}") + continue + meta_dict = dict(meta_dict) # avoid mutating caller input + meta_dict.pop("id", None) + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + return Response( + {"detail": f"serialized value exceeds {SPARSE_FIELD_VALUE_MAX_LENGTH} characters"}, status=400 + ) + cleaned.append(value) + + for value in cleaned: + name = _next_sparse_name(resource) + try: + SparseField.objects.create(resource=resource, name=name, value=value) + except IntegrityError: + # Concurrent request created the same name; retry with UUID + name = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + SparseField.objects.create(resource=resource, name=name, value=value) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result, status=201) + + # -- PUT ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_put(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + cleaned_updates = [] + for meta_dict in data: + if not isinstance(meta_dict, dict): + continue + meta_dict = dict(meta_dict) # avoid mutating caller input + sf_id = meta_dict.pop("id", None) + if sf_id is None: + continue + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + return Response( + {"detail": f"serialized value exceeds {SPARSE_FIELD_VALUE_MAX_LENGTH} characters"}, status=400 + ) + cleaned_updates.append((sf_id, value)) + + for sf_id, value in cleaned_updates: + SparseField.objects.filter( + pk=sf_id, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).update(value=value) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) + + # -- DELETE ------------------------------------------------------------- + + @staticmethod + def _extra_metadata_delete(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of IDs."}, + status=400, + ) + + ids = [int(i) for i in data if isinstance(i, (int, str)) and str(i).isdigit()] + SparseField.objects.filter( + pk__in=ids, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).delete() + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) diff --git a/geonode/base/api/exceptions.py b/geonode/base/api/exceptions.py index 32812298acc..c3e8cf3548b 100644 --- a/geonode/base/api/exceptions.py +++ b/geonode/base/api/exceptions.py @@ -32,14 +32,18 @@ def geonode_exception_handler(exc, context): # to get the standard error response. response = exception_handler(exc, context) - if response is not None and isinstance(exc, APIException): - # for the upload exception we need a custom response - detail = _extract_detail(exc) - response.data = { - "success": False, - "errors": [str(detail)], - "code": exc.code if hasattr(exc, "code") else exc.default_code, - } + if response is not None: + if response.status_code == 401: + response.headers.pop("WWW-Authenticate", None) + + if isinstance(exc, APIException): + # for the upload exception we need a custom response + detail = _extract_detail(exc) + response.data = { + "success": False, + "errors": [str(detail)], + "code": exc.code if hasattr(exc, "code") else exc.default_code, + } return response diff --git a/geonode/base/api/filters.py b/geonode/base/api/filters.py index 15b4607bf23..da20dd21f50 100644 --- a/geonode/base/api/filters.py +++ b/geonode/base/api/filters.py @@ -33,8 +33,10 @@ class DynamicSearchFilter(SearchFilter): + # search_fields from the request are restricted to the view's allow-list def get_search_fields(self, view, request): - return request.GET.getlist("search_fields", []) + allowed = getattr(view, "search_fields", None) or [] + return [f for f in request.GET.getlist("search_fields", []) if f in allowed] class ExtentFilter(BaseFilterBackend): diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index fa1add81264..442fe244dc7 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -27,7 +27,7 @@ from django.contrib.auth.models import Group from django.forms.models import model_to_dict from django.contrib.auth import get_user_model -from django.db.models.query import QuerySet + from geonode.assets.utils import get_default_asset, is_asset_deletable from geonode.metadata.multilang.serializers import MultiLangOutputMixin from geonode.people import Roles @@ -57,7 +57,6 @@ SpatialRepresentationType, ThesaurusKeyword, ThesaurusKeywordLabel, - ExtraMetadata, LinkedResource, ) from geonode.documents.models import Document @@ -68,9 +67,11 @@ from geonode.assets.handlers import asset_handler_registry from geonode.utils import build_absolute_uri from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataField from geonode.resource.models import ExecutionRequest from django.contrib.gis.geos import Polygon from geonode.security.registry import permissions_registry +from geonode.people.utils import contains_disallowed_template_tokens logger = logging.getLogger(__name__) @@ -178,6 +179,18 @@ class Meta: keywords = serializers.SlugRelatedField(many=True, slug_field="slug", read_only=True) categories = serializers.SlugRelatedField(many=True, slug_field="slug", queryset=GroupCategory.objects.all()) + def validate(self, data): + field_errors = { + field_name: "This field contains characters that are not allowed." + for field_name, value in data.items() + if contains_disallowed_template_tokens(value) + } + + if field_errors: + raise serializers.ValidationError(field_errors) + + return data + class SimpleHierarchicalKeywordSerializer(DynamicModelSerializer): class Meta: @@ -282,23 +295,6 @@ def get_attribute(self, instance): return build_absolute_uri(instance.detail_url) -class ExtraMetadataSerializer(DynamicModelSerializer): - class Meta: - model = ExtraMetadata - name = "ExtraMetadata" - fields = ("pk", "metadata") - - def to_representation(self, obj): - if isinstance(obj, QuerySet): - out = [] - for el in obj: - out.append({**{"id": el.id}, **el.metadata}) - return out - elif isinstance(obj, list): - return obj - return {**{"id": obj.id}, **obj.metadata} - - class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -672,7 +668,8 @@ class ResourceBaseSerializer(MultiLangOutputMixin, DynamicModelSerializer): links = DynamicRelationField(LinksSerializer, source="id", read_only=True) # Deferred fields - metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, many=True, deferred=True) + # Deprecated: use the sparse fields API instead of ``metadata``. + metadata = DeprecatedExtraMetadataField(deferred=True, read_only=True) data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False) executions = DynamicRelationField( ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True @@ -754,7 +751,7 @@ class Meta: "sourcetype", "is_copyable", "blob", - "metadata", + "metadata", # Deprecated: use sparse fields API instead "executions", "linked_resources", "download_url", diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 19fc9d49f87..ea8eed8bb27 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -71,7 +71,6 @@ ResourceBase, TopicCategory, ThesaurusKeyword, - ExtraMetadata, RestrictionCodeType, License, Group, @@ -872,6 +871,44 @@ def test_search_resources(self): # Pagination self.assertEqual(len(response.data["resources"]), 1) + def test_search_fields_allow_list(self): + """ + search_fields not allowed by the view are ignored. + """ + term = "no-such-value-b3wf" + + def _check(url, allowed, rejected): + total = self.client.get(url, format="json").data["total"] + self.assertGreater(total, 0) + for field in allowed: + response = self.client.get(f"{url}?search={term}&search_fields={field}", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 0, field) + for field in rejected: + response = self.client.get(f"{url}?search={term}&search_fields={field}", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], total, field) + + resource_rejected = ["owner__password__startswith", "owner__email__regex", "owner__username", "uuid", "^title"] + # anonymous + for name in ("base-resources-list", "datasets-list", "maps-list", "documents-list"): + _check(reverse(name), allowed=["title", "abstract"], rejected=resource_rejected) + + admin = get_user_model().objects.get(username="admin") + GeoApp.objects.create(title="search fields geoapp", owner=admin) + self.assertTrue(self.client.login(username="admin", password="admin")) + _check(reverse("geoapps-list"), allowed=["title", "abstract"], rejected=resource_rejected) + _check( + reverse("users-list"), + allowed=["username", "first_name", "last_name"], + rejected=["password__startswith", "email", "username__regex"], + ) + _check( + reverse("group-profiles-list"), + allowed=["title", "slug"], + rejected=["group__user__password__startswith", "description"], + ) + def test_filter_resources(self): """ Ensure we can filter across the Resource Base list. @@ -2189,6 +2226,29 @@ def test_set_thumbnail_from_bbox_from_Anonymous_user_raise_permission_error(self self.assertEqual(response.status_code, 401) self.assertEqual(expected, response.json()) + def test_set_thumbnail_from_bbox_without_resource_change_permission_rejected(self): + bobby = get_user_model().objects.get(username="bobby") + resource = Dataset.objects.first() + resource.owner = bobby + resource.save() + + self.client.logout() + self.assertTrue(self.client.login(username="norman", password="norman")) + + url = reverse("base-resources-set-thumb-from-bbox", kwargs={"resource_id": resource.id}) + payload = { + "bbox": [-9072629.904175375, -9043966.018568434, 1491839.8773032012, 1507127.2829602365], + "srid": "EPSG:3857", + } + + response = self.client.post(url, data=payload, format="json") + + self.assertEqual(response.status_code, 403) + self.assertEqual( + response.json(), + {"message": "You do not have permission to set this thumbnail.", "success": False}, + ) + @patch("geonode.layers.manager.DatasetResourceManager.set_thumbnail") def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset(self, mock_set_thumbnail): """ @@ -2910,7 +2970,7 @@ def test_resource_service_copy_with_perms_dataset(self): @patch.dict(os.environ, {"ASYNC_SIGNALS": "False"}) @override_settings(ASYNC_SIGNALS=False) - def test_resource_service_copy_with_perms_dataset_set_default_perms(self): + def test_resource_service_copy_with_perms_dataset_keep_original_perms(self): with self.settings(ASYNC_SIGNALS=False): files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) @@ -2951,7 +3011,7 @@ def test_resource_service_copy_with_perms_dataset_set_default_perms(self): self.assertEqual("finished", self.client.get(response.json().get("status_url")).json().get("status")) _resource = Dataset.objects.filter(title__icontains="test_copy_with_perms").last() self.assertIsNotNone(_resource) - self.assertNotIn( + self.assertIn( "bobby", [x.username for x in permissions_registry.get_perms(instance=_resource).get("users", [])], ) @@ -3296,78 +3356,20 @@ def test_metadata_uploaded_preserve_can_be_updated(self): self.assertTrue(doc.metadata_uploaded_preserve) self.assertTrue(response.json()["resource"]["metadata_uploaded_preserve"]) - -class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): - def setUp(self): - self.layer = create_single_dataset("single_layer") - self.metadata = { - "filter_header": "Foo Filter header", - "field_name": "metadata-name", - "field_label": "this is the help text", - "field_value": "foo", - } - m = ExtraMetadata.objects.create(resource=self.layer, metadata=self.metadata) - self.layer.metadata.add(m) - self.mdata = ExtraMetadata.objects.first() - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(200, response.status_code) - expected = [{**{"id": self.mdata.id}, **self.metadata}] - self.assertEqual(expected, response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - input_metadata = { - "id": self.mdata.id, - "filter_header": "Foo Filter header", - "field_name": "metadata-updated", - "field_label": "this is the help text", - "field_value": "foo", - } - response = self.client.put(url, data=[input_metadata], content_type="application/json") - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - input_metadata = { - "filter_header": "Foo Filter header", - "field_name": "metadata-updated", - "field_label": "this is the help text", - "field_value": "foo", - } - response = self.client.post(url, data=[input_metadata], content_type="application/json") - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.delete(url, data=[self.mdata.id], content_type="application/json") - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) - - def test_user_without_view_perms_cannot_see_the_endpoint(self): - from geonode.resource.registry import resource_manager_registry - - self.client.login(username="bobby", password="bob") - resource_manager_registry.get_for_instance(self.layer.get_self_resource()).remove_permissions( - self.layer.uuid, instance=self.layer.get_self_resource() - ) - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(401, response.status_code) - - perm_spec = {"users": {"bobby": ["view_resourcebase"]}, "groups": {}} - self.layer.set_permissions(perm_spec) - url = reverse("base-resources-extra-metadata", args=[self.layer.id]) - response = self.client.get(url, content_type="application/json") - self.assertTrue(200, response.status_code) + def test_www_authenticate_header_is_removed_for_401_responses(self): + """ + Ensure WWW-Authenticate header is removed for 401 responses to prevent browsers from showing a login prompt + """ + try: + user = get_user_model().objects.create_user( + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) + # Anonymous can't read + response = self.client.get(url, format="json") + self.assertNotIn("WWW-Authenticate", response.headers) + finally: + user.delete() class TestApiLinkedResources(GeoNodeBaseTestSupport): @@ -3415,6 +3417,24 @@ def test_insert_one_linked_resource(self): self.assertEqual(self.map.id, link_connected.target_id) + def test_linked_resource_requires_target_view_permission(self): + user = get_user_model().objects.create_user(username="linked_resource_user", password="test") + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + LinkedResource.objects.filter(source=self.doc, target=self.map).delete() + + self.doc.set_permissions( + {"users": {user.username: ["base.view_resourcebase", "base.change_resourcebase"]}, "groups": {}} + ) + self.map.set_permissions({"users": {}, "groups": {}}) + + self.client.force_login(user) + response = self.client.post(url, data={"target": [self.map.id]}, content_type="application/json") + + self.assertEqual(response.status_code, 400) + self.assertIn(self.map.id, response.json()["error"]) + self.assertFalse(LinkedResource.objects.filter(source=self.doc, target=self.map).exists()) + def test_insert_linked_resource_invalid_type(self): url = reverse("base-resources-linked_resources", args=[self.doc.id]) @@ -4462,3 +4482,150 @@ def test_map_layer_permission_caching(self): # Check that the permissions in the layers are the same for layer1, layer2 in zip(data1["map"]["maplayers"], data2["map"]["maplayers"]): self.assertEqual(layer1["dataset"]["perms"], layer2["dataset"]["perms"]) + + +class DeprecatedExtraMetadataApiTest(GeoNodeBaseTestSupport): + """Tests for the deprecated backward-compatible ExtraMetadata API adapters. + + These adapters re-expose the old ``/extra_metadata/`` endpoint and the + ``metadata`` serializer field using SparseField as the storage backend. + """ + + def setUp(self): + super().setUp() + self.admin = get_user_model().objects.get(username="admin") + self.dataset = create_single_dataset("deprecated_em_test") + + def _url(self, pk=None): + pk = pk or self.dataset.pk + return urljoin( + f"{reverse('base-resources-list')}/", + f"{pk}/extra_metadata/", + ) + + def test_get_empty_extra_metadata(self): + """GET should return an empty list when no extra metadata exists.""" + self.client.login(username="admin", password="admin") + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), []) + + def test_post_extra_metadata(self): + """POST should create new extra metadata entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + payload = [ + {"field_name": "test_field", "field_value": "test_value"}, + {"field_name": "another", "field_value": "value2"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 201) + result = response.json() + self.assertEqual(len(result), 2) + # Each entry should have an id and the metadata fields + for item in result: + self.assertIn("id", item) + self.assertIn("field_name", item) + self.assertIn("field_value", item) + + # Verify SparseField entries were created + sf_count = SparseField.objects.filter( + resource=self.dataset.resourcebase_ptr, + name__startswith="extra_", + ).count() + self.assertEqual(sf_count, 2) + + def test_get_returns_posted_metadata(self): + """GET after POST should return the created metadata.""" + self.client.login(username="admin", password="admin") + payload = [{"field_name": "myfield", "field_value": "myvalue"}] + self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "myfield") + self.assertEqual(result[0]["field_value"], "myvalue") + + def test_put_updates_existing_metadata(self): + """PUT should update an existing entry by id.""" + self.client.login(username="admin", password="admin") + # Create first + payload = [{"field_name": "original", "field_value": "v1"}] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + created_id = response.json()[0]["id"] + + # Update + update_payload = [{"id": created_id, "field_name": "updated", "field_value": "v2"}] + response = self.client.put( + self._url(), + data=json.dumps(update_payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "updated") + self.assertEqual(result[0]["field_value"], "v2") + + def test_delete_removes_metadata(self): + """DELETE should remove entries by id.""" + self.client.login(username="admin", password="admin") + # Create + payload = [ + {"field_name": "to_delete", "field_value": "val"}, + {"field_name": "to_keep", "field_value": "keep"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + items = response.json() + delete_id = items[0]["id"] + + # Delete one + response = self.client.delete( + self._url(), + data=json.dumps([delete_id]), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertNotEqual(result[0]["id"], delete_id) + + def test_metadata_field_in_serializer(self): + """The deprecated ``metadata`` field should appear when requested + via include[] and return data from SparseField entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + SparseField.objects.create( + resource=self.dataset.resourcebase_ptr, + name="extra_1", + value=json.dumps({"field_name": "test", "field_value": "val"}), + ) + url = f"{reverse('base-resources-list')}/{self.dataset.pk}?include[]=metadata" + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + data = response.json().get("resource", response.json()) + self.assertIn("metadata", data) + metadata = data["metadata"] + self.assertIsInstance(metadata, list) + if metadata: + self.assertIn("id", metadata[0]) + self.assertIn("field_name", metadata[0]) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 59167c1dd23..f0d0af7944f 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -64,7 +64,6 @@ TopicCategory, ThesaurusKeyword, Configuration, - ExtraMetadata, LinkedResource, ) from geonode.base.api.filters import ( @@ -89,7 +88,6 @@ from geonode.resource.registry import resource_manager_registry from .permissions import ( - IsOwnerOrAdmin, IsManagerEditOrAdmin, ResourceBasePermissionsFilter, UserHasPerms, @@ -104,17 +102,17 @@ TopicCategorySerializer, RegionSerializer, ThesaurusKeywordSerializer, - ExtraMetadataSerializer, LinkedResourceSerializer, ) from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination -from geonode.base.utils import validate_extra_metadata, patch_perms +from geonode.base.utils import patch_perms +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataMixin from geonode.assets.models import Asset from geonode.assets.utils import create_asset_and_link, unlink_asset from geonode.assets.handlers import asset_handler_registry from geonode.utils import get_supported_datasets_file_types - +from geonode.utils import assert_safe_xml, UnsafeXMLError logger = logging.getLogger(__name__) @@ -129,6 +127,7 @@ class GroupViewSet(DynamicModelViewSet): IsManagerEditOrAdmin, ] filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] + search_fields = ["title", "slug"] serializer_class = GroupProfileSerializer pagination_class = GeoNodeApiPagination @@ -291,7 +290,7 @@ def replace_presets(self, request): request.GET._mutable = False -class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicModelViewSet): +class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DeprecatedExtraMetadataMixin, DynamicModelViewSet): """ API endpoint that allows base resources to be viewed or edited. """ @@ -309,6 +308,7 @@ class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicMode FavoriteFilter, ] queryset = ResourceBase.objects.select_related("owner").order_by("-created") + search_fields = ["title", "abstract"] serializer_class = ResourceBaseSerializer pagination_class = GeoNodeApiPagination @@ -655,6 +655,13 @@ def set_thumbnail_from_bbox(self, request, resource_id, *args, **kwargs): try: resource = ResourceBase.objects.get(id=ast.literal_eval(resource_id)) + # Check if the current user has the permissions to set the thumbnail + if not request.user.has_perm("change_resourcebase", resource.get_self_resource()): + return Response( + {"message": "You do not have permission to set this thumbnail.", "success": False}, + status=status.HTTP_403_FORBIDDEN, + ) + map_thumb_from_bbox = False if isinstance(resource.get_real_instance(), Map): map_thumb_from_bbox = True @@ -1222,81 +1229,6 @@ def set_thumbnail(self, request, pk, *args, **kwargs): return Response({"thumbnail_url": resource.thumbnail_url}) return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], - url_path=r"extra_metadata", # noqa - url_name="extra-metadata", - ) - def extra_metadata(self, request, pk, *args, **kwargs): - """Get/Update/Delete/Add extra metadata for resource""" - _obj = get_object_or_404(ResourceBase, pk=pk) - - if request.method == "GET": - # get list of available metadata - queryset = _obj.metadata.all() - _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] - if _filters: - queryset = queryset.filter(**_filters[0]) - return Response(ExtraMetadataSerializer().to_representation(queryset)) - if not request.method == "DELETE": - try: - extra_metadata = validate_extra_metadata(request.data, _obj) - except Exception as e: - return Response(status=500, data=e.args[0]) - - if request.method == "PUT": - """ - update specific metadata. The ID of the metadata is required to perform the update - [ - { - "id": 1, - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - """ - for _m in extra_metadata: - _id = _m.pop("id") - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) - logger.info("metadata updated for the selected resource") - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) - elif request.method == "DELETE": - # delete single metadata - """ - Expect a payload with the IDs of the metadata that should be deleted. Payload be like: - [4, 3] - """ - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) - elif request.method == "POST": - # add new metadata - """ - [ - { - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - """ - for _m in extra_metadata: - new_m = ExtraMetadata.objects.create(resource=_obj, metadata=_m) - new_m.save() - _obj.metadata.add(new_m) - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) - @action( detail=True, methods=["get"], @@ -1361,6 +1293,14 @@ def linked_resources(self, request, pk, *args, **kwargs): for t_id in valid_ids: try: target = get_object_or_404(ResourceBase, pk=t_id) + if not permissions_registry.user_has_perm( + request.user, + target.get_self_resource(), + "view_resourcebase", + include_virtual=True, + ): + error_var.append(t_id) + continue if request.method == "POST": _, created = LinkedResource.objects.get_or_create(source=resource, target=target) @@ -1434,6 +1374,17 @@ def asset(self, request, pk=None, *args, **kwargs): {"message": f"The uploaded file type {file_ext} is not allowed."}, status=status.HTTP_400_BAD_REQUEST, ) + if file_ext in ("xml", "sld"): + try: + assert_safe_xml(file.read()) + except UnsafeXMLError: + logger.warning("XML validation failed for uploaded asset.", exc_info=True) + return Response( + {"message": f"The uploaded {file_ext} file is invalid or unsafe."}, + status=status.HTTP_400_BAD_REQUEST, + ) + finally: + file.seek(0) try: handler = asset_handler_registry.get_default_handler() asset, link = create_asset_and_link( diff --git a/geonode/base/management/commands/thesaurus.py b/geonode/base/management/commands/thesaurus.py index 95802eaf451..c7cfd9d0c1c 100644 --- a/geonode/base/management/commands/thesaurus.py +++ b/geonode/base/management/commands/thesaurus.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from geonode.base.management.command_utils import setup_logger +from geonode.base.management.commands.thesaurus_subcommands.autoload import autoload_thesauri from geonode.base.management.commands.thesaurus_subcommands.dump import ( dump_thesaurus, DUMP_FORMATS, @@ -16,7 +17,8 @@ COMMAND_LIST = "list" COMMAND_DUMP = "dump" COMMAND_LOAD = "load" -COMMANDS = [COMMAND_LIST, COMMAND_LOAD, COMMAND_DUMP] +COMMAND_AUTOLOAD = "autoload" +COMMANDS = [COMMAND_LIST, COMMAND_LOAD, COMMAND_DUMP, COMMAND_AUTOLOAD] class Command(BaseCommand): @@ -41,6 +43,12 @@ def add_arguments(self, parser): choices=ACTIONS, help="Actions to run upon data loading (default: create)", ) + load_group.add_argument( + "--langs", + dest="langs", + action="append", + help="Only import labels for the requested languages; can be repeated", + ) dump_group = parser.add_argument_group('Params for "dump" subcommand') dump_group.add_argument("-o", "--out", nargs="?", help="Full path to the output file to be created") @@ -99,6 +107,8 @@ def handle(self, *args, **options): input_file = options.get("file") action = options.get("action") identifier = options.get("identifier") + lang = options.get("lang") + langs = options.get("langs") or [] if not input_file: raise CommandError("'load' command requires the parameter.") @@ -107,7 +117,10 @@ def handle(self, *args, **options): action = ACTION_CREATE logger.info(f"Missing action param: setting actions as '{action}'") - load_thesaurus(input_file, identifier, action) + load_thesaurus(input_file, identifier, action, default_lang=lang, langs=langs) + + elif subcommand == COMMAND_AUTOLOAD: + autoload_thesauri() else: raise CommandError(f"Unknown subcommand: {subcommand}") diff --git a/geonode/base/management/commands/thesaurus_subcommands/autoload.py b/geonode/base/management/commands/thesaurus_subcommands/autoload.py new file mode 100644 index 00000000000..df2725d6dfa --- /dev/null +++ b/geonode/base/management/commands/thesaurus_subcommands/autoload.py @@ -0,0 +1,39 @@ +import os + +from django.apps import apps + +from geonode.base.management.command_utils import setup_logger +from geonode.base.management.commands.thesaurus_subcommands.load import load_thesaurus, ACTION_UPDATE + +logger = setup_logger() + + +def autoload_thesauri(): + """ + Discover and load all thesauri (.rdf files) found in a `thesauri/` directory + within each installed Django app. Uses the `update` action so existing entries + are updated and new ones are created without duplicates. + """ + loaded = 0 + for app_config in apps.get_app_configs(): + thesauri_dir = os.path.join(app_config.path, "thesauri") + logger.debug(f"Looking for auto thesaurus in app '{app_config.name}' path: {thesauri_dir}") + if not os.path.isdir(thesauri_dir): + continue + try: + rdf_files = [f for f in os.listdir(thesauri_dir) if f.lower().endswith(".rdf")] + except OSError as e: + logger.error( + f"Failed to scan thesauri directory for app '{app_config.name}' at '{thesauri_dir}': {e}", + exc_info=True, + ) + continue + for rdf_file in sorted(rdf_files): + rdf_path = os.path.join(thesauri_dir, rdf_file) + logger.info(f"Autoloading thesaurus from app '{app_config.name}': {rdf_path}") + try: + load_thesaurus(rdf_path, identifier=None, action=ACTION_UPDATE, log_details=False) + loaded += 1 + except Exception as e: + logger.error(f"Failed to load thesaurus '{rdf_path}': {e}", exc_info=True) + logger.info(f"Autoload complete: {loaded} thesaurus file(s) loaded.") diff --git a/geonode/base/management/commands/thesaurus_subcommands/load.py b/geonode/base/management/commands/thesaurus_subcommands/load.py index 09498010a6a..d64d664cf45 100644 --- a/geonode/base/management/commands/thesaurus_subcommands/load.py +++ b/geonode/base/management/commands/thesaurus_subcommands/load.py @@ -45,13 +45,16 @@ FAKE_BASE_URI = "http://automatically/added/uri/" -def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): +def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE, default_lang: str = None, langs: List[str] = None, log_details=True): g = Graph() # if the input_file is an UploadedFile object rather than a file path the Graph.parse() # method may not have enough info to correctly guess the type; in this case supply the # name, which should include the extension, to guess_format manually... + # explodes list of comma separated langs into single list of langs + langs = [lang.strip() for item in (langs or []) for lang in item.split(",") if lang.strip()] + filename = input_file.name if isinstance(input_file, UploadedFile) else input_file rdf_format = guess_format(filename) if not identifier: @@ -65,7 +68,7 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): if scheme is None: raise CommandError("ConceptScheme not found in file") - default_lang = getattr(settings, "THESAURUS_DEFAULT_LANG", None) + default_lang = default_lang or getattr(settings, "THESAURUS_DEFAULT_LANG", None) or getattr(settings, "LANGUAGE_CODE", 'en') available_titles = [t for t in itertools.chain(g.objects(scheme, DC.title), g.objects(scheme, DCTERMS.title)) @@ -81,15 +84,21 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): Thesaurus, {"identifier": identifier}, {"date": date_issued, "description": description, "title": thesaurus_title, "about": str(scheme)}, - {"card_min": 0, "card_max": 0, "facet": False} + {"card_min": 0, "card_max": 0, "facet": False}, + log_details ) - tl_cnt = tl_add = 0 + tl_cnt = tl_add = tl_skp = 0 tk_cnt = tk_add = 0 - tkl_cnt = tkl_add = 0 + tkl_cnt = tkl_add = tkl_skp = 0 for lang in available_titles: if lang.language is not None: + tl_cnt += 1 + if langs and lang.language not in langs: + logger.debug(f"Skipping thesaurus label for language '{lang.language}' not in requested langs {langs}") + tl_skp += 1 + continue thesaurus_label, c = _run_action( action, ThesaurusLabel, @@ -99,8 +108,8 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): }, {"label": lang.value}, {}, + log_details ) - tl_cnt += 1 tl_add += 1 if c else 0 for concept in g.subjects(RDF.type, SKOS.Concept): @@ -115,7 +124,8 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): available_labels = [t for t in g.objects(concept, SKOS.prefLabel) if isinstance(t, Literal)] alt_label = value_for_language(available_labels, default_lang) or about - logger.info(f" - Parsed Concept -> about:'{about}' alt:'{alt_label}' pref:'{str(pref)}' ") + if log_details: + logger.info(f" - Parsed Concept -> about:'{about}' alt:'{alt_label}' pref:'{str(pref)}' ") tk, c = _run_action( action, @@ -126,14 +136,21 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): }, {"alt_label": alt_label}, {}, + log_details ) tk_cnt += 1 tk_add += 1 if c else 0 for _, pref_label in preferredLabel(g, concept): + tkl_cnt += 1 lang = pref_label.language + if langs and lang not in langs: + logger.debug(f"Skipping label for language '{lang}' not in requested langs {langs}") + tkl_skp += 1 + continue label = str(pref_label) - logger.info(f" - Label {lang}: {label}") + if log_details: + logger.info(f" - Label {lang}: {label}") tkl, c = _run_action( action, @@ -144,17 +161,17 @@ def load_thesaurus(input_file, identifier: str, action: str = ACTION_CREATE): }, {"label": label}, {}, + log_details ) - tkl_cnt += 1 tkl_add += 1 if c else 0 - logger.warning(f"Thesaurus added: {cr_t}") - logger.warning(f"ThesaurusLabel added: {tl_add:3}/{tl_cnt:3}") - logger.warning(f"ThesaurusKeyword added: {tk_add:3}/{tk_cnt:3}") - logger.warning(f"ThesaurusKeywordLabel added: {tkl_add:3}/{tkl_cnt:3}") + logger.warning(f"Thesaurus added: {cr_t}") + logger.warning(f"ThesaurusLabel: found: {tl_cnt:3} - added: {tl_add:3} - skipped: {tl_skp:3}") + logger.warning(f"ThesaurusKeyword: found: {tk_cnt:3} - added: {tk_add:3}") + logger.warning(f"ThesaurusKeywordLabel: found: {tkl_cnt:3} - added: {tkl_add:3} - skipped: {tkl_skp:3}") -def _run_action(action: str, model: type[models.Model], pk_dict, upd_dict, create_dict) -> tuple[models.Model, bool]: +def _run_action(action: str, model: type[models.Model], pk_dict, upd_dict, create_dict, log_details) -> tuple[models.Model, bool]: def update_or_create(defaults=upd_dict, create_defaults=create_dict, **pk_dict): # this signature is available since django 5 obj, created = model.objects.get_or_create(defaults=upd_dict | create_dict, **pk_dict) @@ -162,7 +179,8 @@ def update_or_create(defaults=upd_dict, create_defaults=create_dict, **pk_dict): if not created: rows = model.objects.filter(pk=obj.pk).update(**upd_dict) if rows != 1: - logger.error(f"UPDATED {rows} rows for {model.__name__} -> {pk_dict}") + if log_details: + logger.error(f"UPDATED {rows} rows for {model.__name__} -> {pk_dict}") return obj, created @@ -176,14 +194,17 @@ def update_or_create(defaults=upd_dict, create_defaults=create_dict, **pk_dict): elif action == ACTION_UPDATE: obj, created = update_or_create(defaults=upd_dict, create_defaults=create_dict, **pk_dict) if created: - logger.info(f"{model.__name__} -> Created id:{pk_dict}") + if log_details: + logger.info(f"{model.__name__} -> Created id:{pk_dict}") else: - logger.info(f"{model.__name__} -> Updated id:{pk_dict} DATA:{upd_dict}") + if log_details: + logger.info(f"{model.__name__} -> Updated id:{pk_dict} DATA:{upd_dict}") elif action == ACTION_APPEND: obj, created = model.objects.get_or_create(defaults=upd_dict | create_dict, **pk_dict) if created: - logger.info(f"{model.__name__} -> Created {pk_dict}") + if log_details: + logger.info(f"{model.__name__} -> Created {pk_dict}") else: raise CommandError("No valid action found") diff --git a/geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py b/geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py new file mode 100644 index 00000000000..7d4667d6b77 --- /dev/null +++ b/geonode/base/migrations/0100_migrate_extrametadata_to_sparsefields.py @@ -0,0 +1,52 @@ +import json +import logging + +from django.db import migrations + +logger = logging.getLogger(__name__) + + +def migrate_extrametadata_to_sparsefields(apps, schema_editor): + """ + Migrate ExtraMetadata to SparseField. + Each ExtraMetadata object (with its JSON dict) is stored as a single + SparseField entry with name 'extra_' and value as JSON string. + Entries whose serialized JSON exceeds 1024 characters are skipped with a warning. + """ + ExtraMetadata = apps.get_model("base", "ExtraMetadata") + SparseField = apps.get_model("metadata", "SparseField") + + for extra_meta in ExtraMetadata.objects.select_related("resource").iterator(): + name = f"extra_{extra_meta.pk}" + value = json.dumps(extra_meta.metadata) + if len(value) > 1024: + logger.warning( + f"ExtraMetadata pk={extra_meta.pk} for resource {extra_meta.resource.id}:{extra_meta.resource.title} " + f"skipped during migration to SparseField: " + f"serialized value exceeds 1024 characters" + ) + continue + SparseField.objects.get_or_create( + resource=extra_meta.resource, + name=name, + defaults={"value": value}, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0099_resourcebase_auth_config"), + ("metadata", "0001_initial"), + ] + + operations = [ + migrations.RunPython(migrate_extrametadata_to_sparsefields, migrations.RunPython.noop), + migrations.RemoveField( + model_name="resourcebase", + name="metadata", + ), + migrations.DeleteModel( + name="ExtraMetadata", + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index a04830adc2b..90850fe7f16 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -679,9 +679,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): data_quality_statement_help_text = _( "general explanation of the data producer's knowledge about the lineage of a" " dataset" ) - extra_metadata_help_text = _( - 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]' - ) # internal fields uuid = models.CharField(max_length=36, unique=True, default=uuid.uuid4) title = models.CharField(_("title"), max_length=255, help_text=_("name by which the cited resource is known")) @@ -894,10 +891,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): subtype = models.CharField(max_length=128, null=True, blank=True) - metadata = models.ManyToManyField( - "ExtraMetadata", verbose_name=_("Extra Metadata"), null=True, blank=True, help_text=extra_metadata_help_text - ) - objects = ResourceBaseManager() class Meta: @@ -2174,8 +2167,3 @@ class GroupGeoLimit(models.Model): group = models.ForeignKey(GroupProfile, null=False, blank=False, on_delete=models.CASCADE) resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) wkt = models.TextField(db_column="wkt", blank=True) - - -class ExtraMetadata(models.Model): - resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) - metadata = JSONField(null=True, default=dict, blank=True) diff --git a/geonode/base/utils.py b/geonode/base/utils.py index c73623ce146..91df3069bfb 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -22,16 +22,12 @@ # Standard Modules import re -import json import logging -from schema import Schema from dateutil.parser import isoparse from datetime import datetime, timedelta # Django functionality -from django.conf import settings from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError # Geonode functionality from geonode.layers.models import Dataset @@ -177,33 +173,6 @@ def get_resource(resource_base): return resource_base.get_real_instance() -def validate_extra_metadata(data, instance): - if not data: - return data - - # starting validation of extra metadata passed via JSON - # if schema for metadata validation is not defined, an error is raised - resource_type = instance.polymorphic_ctype.model if instance.polymorphic_ctype else instance.class_name.lower() - extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) - if not extra_metadata_validation_schema: - raise ValidationError(f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}") - # starting json structure validation. The Field can contain multiple metadata - try: - if isinstance(data, str): - data = json.loads(data) - except Exception: - raise ValidationError("The value provided for the Extra metadata field is not a valid JSON") - - # looping on all the single metadata provided. If it doen't match the schema an error is raised - for _index, _metadata in enumerate(data): - try: - Schema(extra_metadata_validation_schema).validate(_metadata) - except Exception as e: - raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") - # conerted because in this case, we can store a well formated json instead of the user input - return data - - def remove_country_from_languagecode(language: str): """Remove country code (us) from language name (en-us) >>> remove_country_from_lanugecode("en-us") diff --git a/geonode/base/views.py b/geonode/base/views.py index 19c30ac94b9..79a14eb0006 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -18,8 +18,6 @@ ######################################################################### import json import logging -import ast - from dal import views, autocomplete from guardian.shortcuts import get_objects_for_user @@ -43,7 +41,6 @@ from geonode.utils import resolve_object from geonode.groups.models import GroupProfile from geonode.tasks.tasks import set_permissions -from geonode.resource.registry import resource_manager_registry from geonode.security.utils import get_visible_resources from geonode.notifications_helper import send_notification from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_languagecode @@ -56,6 +53,7 @@ from geonode.security.views import _perms_info_json from geonode.security.registry import permissions_registry +from django.views.decorators.http import require_GET logger = logging.getLogger(__name__) @@ -344,6 +342,7 @@ def _resolve_resourcebase(request, id, permission="base.change_resourcebase", ms return resolve_object(request, ResourceBase, {"pk": id}, permission=permission, permission_msg=msg, **kwargs) +@require_GET @xframe_options_sameorigin def resourcebase_embed(request, resourcebaseid, template="base/base_edit.html"): """ @@ -373,15 +372,6 @@ def resourcebase_embed(request, resourcebaseid, template="base/base_edit.html"): group = None r = resourcebase_obj - if request.method in ("POST", "PATCH", "PUT"): - resolved_resource_manager = resource_manager_registry.get_for_instance(resourcebase_obj) - r = resolved_resource_manager.update(resourcebase_obj.uuid, instance=resourcebase_obj, notify=True) - - resolved_resource_manager.set_permissions( - resourcebase_obj.uuid, instance=resourcebase_obj, permissions=ast.literal_eval(permissions_json) - ) - - resolved_resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False) access_token = None if request and request.user: diff --git a/geonode/br/management/commands/create_tile_layers.py b/geonode/br/management/commands/create_tile_layers.py index 57b81d805aa..1b40295806e 100644 --- a/geonode/br/management/commands/create_tile_layers.py +++ b/geonode/br/management/commands/create_tile_layers.py @@ -19,155 +19,45 @@ ######################################################################### import logging -import requests -from requests.auth import HTTPBasicAuth - from django.core.management.base import BaseCommand -from django.conf import settings - -from geonode.layers.models import Dataset +from typing_extensions import deprecated +from geonode.geoserver.management.commands.gwc_subcommands.create import CreateTileLayers as gwc_create logger = logging.getLogger(__name__) -REQ_TEMPLATE = """ - - true - true - {} - - 2 - 1 - - - application/json;type=utfgrid - image/gif - image/jpeg - image/png - image/png8 - image/vnd.jpeg-png - image/vnd.jpeg-png8 - - - - EPSG:3857 - - - EPSG:3857x2 - - - EPSG:4326 - - - EPSG:4326x2 - - - EPSG:900913 - - - 0 - 0 - true - 0 - -""" - - +@deprecated("THIS COMMAND IS DEPRECATED - USE THE gwc COMMAND INSTEAD") class Command(BaseCommand): - help = "Create missing TileLayers in GWC" + help = "DEPRECATED COMMAND - USE gwc COMMAND INSTEAD - Create missing TileLayers in GWC" def add_arguments(self, parser): parser.add_argument( - '-f', - '--force', + "-f", + "--force", dest="force", - action='store_true', - help="Force tile layer re-creation also if it already exists in GWC") + action="store_true", + help="Force tile layer re-creation also if it already exists in GWC", + ) - parser.add_argument( - '-l', - '--layer', - dest="layers", - action='append', - help="Only process specified layers ") + parser.add_argument("-l", "--layer", dest="layers", action="append", help="Only process specified layers ") parser.add_argument( - '-d', - '--dry-run', - dest="dry-run", - action='store_true', - help="Do not actually perform any change on GWC") + "-d", "--dry-run", dest="dry-run", action="store_true", help="Do not actually perform any change on GWC" + ) def handle(self, **options): - force = options.get('force') - requested_layers = options.get('layers') - dry_run = options.get('dry-run') + logger.error("THIS COMMAND IS DEPRECATED - USE THE gwc COMMAND INSTEAD") + + force = options.get("force") + requested_layers = options.get("layers") or [] + dry_run = options.get("dry-run") logger.debug(f"FORCE is {force}") logger.debug(f"DRY-RUN is {dry_run}") logger.debug(f"LAYERS is {requested_layers}") - try: - baseurl = settings.OGC_SERVER["default"]["LOCATION"] - user = settings.OGC_SERVER["default"]["USER"] - passwd = settings.OGC_SERVER["default"]["PASSWORD"] - """ - curl -v -u admin:geoserver -XGET \ - "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" - """ - layers = Dataset.objects.all() - tot = len(layers) - logger.info(f"Total layers in GeoNode: {tot}") - i = 0 - cnt_old = 0 - cnt_new = 0 - cnt_bad = 0 - cnt_skip = 0 - cnt_force = 0 - for layer in layers: - i += 1 - logger.info(f"- {i}/{tot} Processing layer: {layer.typename}") - - if requested_layers and layer.typename not in requested_layers: - logger.info(" - Layer filtered out by args") - cnt_skip += 1 - continue - - r = requests.get(f"{baseurl}gwc/rest/layers/{layer.typename}.xml", auth=HTTPBasicAuth(user, passwd)) - - if r.status_code == 200: - if force: - logger.info(" - Forcing layer configuration in GWC") - cnt_force += 1 - else: - logger.info(" - Layer already configured in GWC") - cnt_old += 1 - continue - try: - data = REQ_TEMPLATE.format(layer.name) - url = f"{baseurl}gwc/rest/layers/{layer.typename}.xml" - logger.info(" - Configuring...") - - if not dry_run: - response = requests.put( - url, data=data, headers={"Content-Type": "text/xml"}, auth=HTTPBasicAuth(user, passwd) - ) - - if dry_run or response.status_code == 200: - logger.info(f" - Done {layer.name}") - cnt_new += 1 - else: - logger.warning(f"Layer {layer.typename} couldn't be configured: code {response.status_code}") - cnt_bad += 1 - - except Exception as e: - raise e - except Exception as e: - raise e - - logger.info("Work completed") - logger.info(f"- TileLayers configured: {cnt_new}" + (f" (forced {cnt_force})" if cnt_force else "")) - logger.info(f"- TileLayers in error : {cnt_bad}") - logger.info(f"- TileLayers untouched : {cnt_old}") - logger.info(f"- TileLayers skipped : {cnt_skip}") + subcommand = gwc_create() + subcommand.run( + force=force, requested_layers=requested_layers, create_all=len(requested_layers) == 0, dry_run=dry_run + ) diff --git a/geonode/catalogue/models.py b/geonode/catalogue/models.py index e575be179e6..c268b82c315 100644 --- a/geonode/catalogue/models.py +++ b/geonode/catalogue/models.py @@ -23,6 +23,7 @@ from django.db.models import signals from lxml import etree from owslib.etree import etree as dlxml +from polymorphic.models import PolymorphicTypeInvalid from geonode.layers.models import Dataset from geonode.documents.models import Document from geonode.catalogue import get_catalogue @@ -41,7 +42,6 @@ def catalogue_pre_delete(instance, sender, **kwargs): def catalogue_post_save(instance, sender, **kwargs): """Get information from catalogue""" _id = instance.resourcebase_ptr.id if hasattr(instance, "resourcebase_ptr") else instance.id - resources = ResourceBase.objects.filter(id=_id) # Update the Catalog try: @@ -49,40 +49,54 @@ def catalogue_post_save(instance, sender, **kwargs): catalogue.create_record(instance) record = catalogue.get_record(instance.uuid) except OSError as err: - msg = f'Could not connect to catalogue to save information for layer "{instance.name}"' if err.errno == errno.ECONNREFUSED: - LOGGER.warn(msg, err) + LOGGER.warning( + f'Could not connect to catalogue to save information for layer "{instance.name}": {err}', exc_info=err + ) return else: - raise err + raise - if not record: - msg = f"Metadata record for {instance.title} does not exist, check the catalogue signals." - LOGGER.warning(msg) - return + if record: + _recreate_links(instance, record) + else: + LOGGER.warning(f"Metadata record for {instance.title} does not exist, check the catalogue signals.") + + ResourceBase.objects.filter(id=_id).update(csw_wkt_geometry=instance.geographic_bounding_box) + update_csw_metadata(instance, catalogue) + +def _recreate_links(instance, record): if not hasattr(record, "links"): msg = f"Metadata record for {instance.title} should contain links." raise Exception(msg) # Create the different metadata links with the available formats - if resources.exists(): - for mime, name, metadata_url in record.links["metadata"]: - try: - Link.objects.get_or_create( - resource=resources.get(), - url=metadata_url, - defaults=dict(name=name, extension="xml", mime=mime, link_type="metadata"), - ) - except Exception: - _d = dict(name=name, extension="xml", mime=mime, link_type="metadata") - Link.objects.filter( - resource=resources.get(), url=metadata_url, extension="xml", link_type="metadata" - ).update(**_d) + for mime, name, metadata_url in record.links["metadata"]: + try: + Link.objects.get_or_create( + resource_id=instance.id, + url=metadata_url, + defaults=dict(name=name, extension="xml", mime=mime, link_type="metadata"), + ) + except Exception: + _d = dict(name=name, extension="xml", mime=mime, link_type="metadata") + Link.objects.filter( + resource_id=instance.id, url=metadata_url, extension="xml", link_type="metadata" + ).update(**_d) + + +def update_csw_metadata(instance, catalogue=None): + if not catalogue: + catalogue = get_catalogue() if instance.metadata_uploaded and instance.metadata_uploaded_preserve: md_doc = etree.tostring(dlxml.fromstring(instance.metadata_xml)) else: + try: + instance = instance.get_real_instance() # get as many attrs as possible from the instance + except PolymorphicTypeInvalid as e: + LOGGER.warning(f"Could not resolve real instance for {instance.title}, using it as-is: {e}") # generate an XML document (GeoNode's default is ISO) raw_xml = catalogue.catalogue.csw_gen_xml(instance, settings.CATALOG_METADATA_TEMPLATE) md_obj = dlxml.fromstring(raw_xml, parser=etree.XMLParser(remove_blank_text=True)) @@ -91,10 +105,10 @@ def catalogue_post_save(instance, sender, **kwargs): try: csw_anytext = catalogue.catalogue.csw_gen_anytext(md_doc) except Exception as e: - LOGGER.exception(e) + LOGGER.exception(f"Error while generating ANYTEXT: {e}", exc_info=e) csw_anytext = "" - resources.update(metadata_xml=md_doc, csw_wkt_geometry=instance.geographic_bounding_box, csw_anytext=csw_anytext) + ResourceBase.objects.filter(pk=instance.id).update(metadata_xml=md_doc, csw_anytext=csw_anytext) if "geonode.catalogue" in settings.INSTALLED_APPS: diff --git a/geonode/catalogue/templates/catalogue/full_metadata.xml b/geonode/catalogue/templates/catalogue/full_metadata.xml index d54c45a59a7..0cb7cbd65f5 100644 --- a/geonode/catalogue/templates/catalogue/full_metadata.xml +++ b/geonode/catalogue/templates/catalogue/full_metadata.xml @@ -1,11 +1,14 @@ {% load thesaurus %} {% load l10n %} +{% load multilang %} {{layer.uuid}} + {% language_info layer.language as resource_language %} + {% languages_info metadata as locales %} - {{layer.language}} + {{ resource_language.label }} utf8 @@ -91,6 +94,18 @@ ISO 19115:2003 + {% for locale in locales %} + + + + {{ locale.label }} + + + {{ locale.encoding }} + + + + {% endfor %} @@ -114,8 +129,18 @@ - + {% multilang_values "title" metadata as title_translations %} + {{layer.title}} + {% if title_translations %} + + {% for t in title_translations %} + + {{ t.text }} + + {% endfor %} + + {% endif %} {% if layer.alternate %} @@ -149,8 +174,18 @@ - + {% multilang_values "abstract" metadata as abstract_translations %} + {{layer.raw_abstract}} + {% if abstract_translations %} + + {% for t in abstract_translations %} + + {{ t.text }} + + {% endfor %} + + {% endif %} {% if layer.raw_purpose %}{{layer.raw_purpose}}{% endif %} @@ -370,7 +405,7 @@ {% endif %} - {{layer.language}} + {{ resource_language.label }} utf8 diff --git a/geonode/catalogue/templatetags/__init__.py b/geonode/catalogue/templatetags/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/catalogue/templatetags/multilang.py b/geonode/catalogue/templatetags/multilang.py new file mode 100644 index 00000000000..ba471ee79a4 --- /dev/null +++ b/geonode/catalogue/templatetags/multilang.py @@ -0,0 +1,158 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django import template +from django.conf import settings + +from geonode.metadata.multilang.utils import ( + get_default_language, + get_3_from_2, + get_2_from_3, + get_multilang_field_names, + get_all_multilang_fields, +) + +register = template.Library() +logger = logging.getLogger(__name__) + + +def _language_label(language_code_2, fallback): + """ + Return the display label for a configured language. + """ + language_labels = {code.split("-")[0]: label for code, label in settings.LANGUAGES} + + return language_labels.get(language_code_2, fallback) + + +def _language_descriptor(lang_2, lang_3): + """Build the language descriptor used by the ISO template.""" + return { + "id": f"locale-{lang_2}", + "iso639_2": lang_3, + "label": _language_label(lang_2, lang_3), + "encoding": "utf8", + } + + +@register.filter(name="is_multilang") +def is_multilang(field_name): + """Return whether the field is configured as multilingual.""" + return field_name in getattr(settings, "MULTILANG_FIELDS", ()) + + +@register.simple_tag(name="multilang_values") +def multilang_values(field_name, metadata): + """ + Return all localized values for a multilingual field. + """ + if not isinstance(metadata, dict) or not is_multilang(field_name): + return [] + + values = [] + + for language_code, translated_field in get_multilang_field_names(field_name): + translated_text = metadata.get(translated_field) + if not translated_text: + continue + + if get_3_from_2(language_code) is None: + logger.warning( + "No entry in LANGUAGE_MAPPINGS for language '%s'; skipping localized value for field '%s'.", + language_code, + field_name, + ) + continue + + values.append( + { + "locale": language_code, + "text": translated_text, + } + ) + + return values + + +@register.simple_tag(name="language_info") +def language_info(lang_3): + """ + Return information about a single ISO639-2 language. + """ + if not lang_3: + lang_2 = get_default_language() + lang_3 = get_3_from_2(lang_2) + if lang_3 is None: + logger.warning( + "No ISO639-2 language found for '%s'; using 'eng' as the default.", + lang_2, + ) + lang_3 = "und" + elif len(lang_3) == 2: + lang_2 = lang_3 + lang_3 = get_3_from_2(lang_2) or "und" + else: + lang_2 = get_2_from_3(lang_3) + + if lang_2 is None: + logger.warning( + "No ISO639-1 language found for '%s'; using the ISO639-2 code as the label.", + lang_3, + ) + + return _language_descriptor( + lang_2, + lang_3, + ) + + +@register.simple_tag(name="languages_info") +def languages_info(metadata, fields=None): + """ + Return descriptors for all translated languages present in the metadata. + """ + if not isinstance(metadata, dict): + return [] + + multilingual_fields = set(fields or settings.MULTILANG_FIELDS) + + languages = set() + + for (field_name, language_code), multilang_field_name in get_all_multilang_fields().items(): + if field_name not in multilingual_fields or not metadata.get(multilang_field_name): + continue + + languages.add(language_code) + + descriptors = [] + for language_code in sorted(languages): + lang_3 = get_3_from_2(language_code) + + if lang_3 is None: + logger.warning( + "No entry in LANGUAGE_MAPPINGS for language '%s'; skipping this entry.", + language_code, + ) + continue + + descriptors.append(_language_descriptor(language_code, lang_3)) + + return descriptors diff --git a/geonode/catalogue/tests.py b/geonode/catalogue/tests.py index 139b2104553..db6d2bd976f 100644 --- a/geonode/catalogue/tests.py +++ b/geonode/catalogue/tests.py @@ -21,11 +21,12 @@ from django.db.models import Q from django.template.loader import get_template -from django.test import RequestFactory +from django.test import RequestFactory, override_settings from django.http.response import Http404 from django.core.exceptions import PermissionDenied from geonode.layers.models import Dataset from geonode.catalogue import get_catalogue +from geonode.catalogue.templatetags import multilang as tags from django.contrib.auth import get_user_model from django.contrib.auth.models import AnonymousUser @@ -33,7 +34,12 @@ from geonode.tests.base import GeoNodeBaseTestSupport from geonode.catalogue.models import catalogue_post_save -from geonode.catalogue.views import csw_global_dispatch, resolve_uuid +from geonode.catalogue.views import ( + csw_global_dispatch, + resolve_uuid, + csw_render_extra_format_html, + csw_render_extra_format_txt, +) from geonode.layers.populate_datasets_data import create_dataset_data from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset @@ -208,3 +214,145 @@ def test_localized_keyword_in_metadata(self): rendered_xml = template.render(context) self.assertIn(self.keyword_label.label, rendered_xml) self.assertNotIn(self.keyword.alt_label, rendered_xml) + + def test_csw_extra_format_html_denies_anonymous_private_resource(self): + self.dataset.set_permissions( + {"groups": {"registered-members": ["base.view_resourcebase", "base.download_resourcebase"]}} + ) + + request = RequestFactory().get(f"/catalogue/csw_to_extra_format/{self.dataset.uuid}/{self.dataset.name}.html") + request.user = AnonymousUser() + + with self.assertRaises(PermissionDenied): + csw_render_extra_format_html(request, self.dataset.uuid, self.dataset.name) + + def test_csw_extra_format_txt_denies_anonymous_private_resource(self): + self.dataset.set_permissions( + {"groups": {"registered-members": ["base.view_resourcebase", "base.download_resourcebase"]}} + ) + + request = RequestFactory().get(f"/catalogue/csw_to_extra_format/{self.dataset.uuid}/{self.dataset.name}.txt") + request.user = AnonymousUser() + + with self.assertRaises(PermissionDenied): + csw_render_extra_format_txt(request, self.dataset.uuid, self.dataset.name) + + +@override_settings(MULTILANG_FIELDS=["title", "abstract"]) +class IsMultilangFilterTest(GeoNodeBaseTestSupport): + + def test_configured_field(self): + self.assertTrue(tags.is_multilang("title")) + self.assertTrue(tags.is_multilang("abstract")) + + def test_unconfigured_field(self): + self.assertFalse(tags.is_multilang("purpose")) + + @override_settings(MULTILANG_FIELDS=()) + def test_feature_off(self): + self.assertFalse(tags.is_multilang("title")) + + +@override_settings(MULTILANG_FIELDS=["title", "abstract"]) +class MultilangValuesTagTests(GeoNodeBaseTestSupport): + + def test_field_not_configured(self): + metadata = {"purpose": "x", "purpose_multilang_it": "scopo"} + self.assertEqual([], tags.multilang_values("purpose", metadata)) + + def test_configured_field_no_translations(self): + self.assertEqual([], tags.multilang_values("title", {"title": "Hello"})) + + def test_excludes_empty_string_translation(self): + metadata = { + "title": "Hello", + "title_multilang_de": "", + "title_multilang_it": "Ciao", + } + self.assertEqual([{"locale": "it", "text": "Ciao"}], tags.multilang_values("title", metadata)) + + def test_multiple_translations(self): + metadata = { + "title": "Hello", + "title_multilang_de": "Hallo", + "title_multilang_es": "Hola", + "title_multilang_fr": "Bonjour", + "title_multilang_it": "Ciao", + } + result = {t["locale"]: t["text"] for t in tags.multilang_values("title", metadata)} + self.assertEqual({"de": "Hallo", "es": "Hola", "fr": "Bonjour", "it": "Ciao"}, result) + + def test_empty_or_missing_metadata(self): + self.assertEqual([], tags.multilang_values("title", {})) + self.assertEqual([], tags.multilang_values("title", None)) + + +@override_settings(MULTILANG_FIELDS=["title", "abstract"]) +class LanguageTemplateTagTests(GeoNodeBaseTestSupport): + + def test_language_info_known_code(self): + result = tags.language_info("ita") + self.assertEqual( + { + "id": "locale-it", + "iso639_2": "ita", + "label": "Italiano", + "encoding": "utf8", + }, + result, + ) + + def test_language_info_code_with_bibliographic_terminological_variant(self): + result = tags.language_info("fra") + self.assertEqual("locale-fr", result["id"]) + self.assertEqual("fra", result["iso639_2"]) + self.assertEqual("Français", result["label"]) + + def test_languages_info_empty_or_missing_metadata(self): + self.assertEqual([], tags.languages_info({})) + self.assertEqual([], tags.languages_info(None)) + + def test_languages_info_no_translations(self): + metadata = {"title": "Hello", "abstract": "An abstract"} + self.assertEqual([], tags.languages_info(metadata)) + + def test_languages_info_across_fields(self): + metadata = { + "title": "Hello", + "title_multilang_it": "Ciao", + "title_multilang_de": "Hallo", + "abstract": "An abstract", + "abstract_multilang_fr": "Un résumé", + "abstract_multilang_de": "Eine Zusammenfassung", + } + result = {d["id"]: d for d in tags.languages_info(metadata)} + self.assertEqual({"locale-it", "locale-de", "locale-fr"}, set(result.keys())) + self.assertEqual( + { + "id": "locale-it", + "iso639_2": "ita", + "label": "Italiano", + "encoding": "utf8", + }, + result["locale-it"], + ) + + def test_languages_info_respects_explicit_fields_argument(self): + metadata = { + "title": "Hello", + "title_multilang_it": "Ciao", + "abstract": "x", + "abstract_multilang_fr": "Un résumé", + } + result = tags.languages_info(metadata, fields=["title"]) + self.assertEqual(["locale-it"], [d["id"] for d in result]) + + @override_settings(LANGUAGE_MAPPINGS=(("en", "eng"), ("it", "ita"))) + def test_languages_info_missing_iso_mapping_is_skipped_and_warns(self): + metadata = { + "title": "Hello", + "title_multilang_it": "Ciao", + "title_multilang_de": "Hallo", + } + result = tags.languages_info(metadata) + self.assertEqual(["locale-it"], [d["id"] for d in result]) diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py index 2d6b6dbae57..00da3ce4a9d 100644 --- a/geonode/catalogue/views.py +++ b/geonode/catalogue/views.py @@ -220,7 +220,12 @@ def get_keywords(resource): @csrf_exempt def csw_render_extra_format_txt(request, layeruuid, resname): """pycsw wrapper""" - resource = ResourceBase.objects.get(uuid=layeruuid) + resource = resolve_object( + request, + ResourceBase, + {"uuid": layeruuid}, + permission="base.view_resourcebase", + ) chrs = get_CSV_spec_char() s = chrs["separator"] c = chrs["carriage_return"] @@ -299,7 +304,12 @@ def __append_contact_role__(content, cr_attr_name, title_in_txt): def csw_render_extra_format_html(request, layeruuid, resname): - resource = ResourceBase.objects.get(uuid=layeruuid) + resource = resolve_object( + request, + ResourceBase, + {"uuid": layeruuid}, + permission="base.view_resourcebase", + ) extra_res_md = {} try: sprt = SpatialRepresentationType.objects.get(id=resource.spatial_representation_type_id) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index fa8d7739bab..c42bfaa444a 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -63,6 +63,7 @@ class DocumentViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicModelVie DocumentPermissionsFilter, ] queryset = Document.objects.all().order_by("-created") + search_fields = ["title", "abstract"] serializer_class = DocumentSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/documents/enumerations.py b/geonode/documents/enumerations.py index 3c592bd3b54..589b370a746 100644 --- a/geonode/documents/enumerations.py +++ b/geonode/documents/enumerations.py @@ -93,7 +93,7 @@ { "csv": {"text/plain", "text/csv"}, "log": {"text/plain", "text/csv"}, - "xml": {"application/xml", "text/xml"}, + "xml": {"application/xml", "text/xml", "text/plain"}, "sld": {"text/plain", "application/xml", "text/xml"}, "dxf": {"image/vnd.dwg", "image/vnd.dxf"}, "doc": {"application/msword", "application/x-ole-storage"}, diff --git a/geonode/documents/forms.py b/geonode/documents/forms.py index 62b27562638..fe39b8646c1 100644 --- a/geonode/documents/forms.py +++ b/geonode/documents/forms.py @@ -30,9 +30,13 @@ from geonode.documents.models import Document from geonode.upload.models import UploadSizeLimit from geonode.upload.api.exceptions import FileUploadLimitException +from geonode.upload.zip_validation import ZipValidationError, is_zip_extension, validate_safe_zip +from geonode.utils import assert_safe_xml, UnsafeXMLError logger = logging.getLogger(__name__) +XML_LIKE_DOCUMENT_EXTENSIONS = {"xml", "sld"} + class SizeRestrictedFileField(forms.FileField): """ @@ -141,4 +145,35 @@ def clean_doc_file(self): logger.debug("This file type is not allowed") raise forms.ValidationError(_("This file type is not allowed")) + # Mirror the importer's zip-safety gate: inspect the central directory + # of any zip-based document upload (.zip plus OOXML / ODF formats) to + # reject path-traversal entries, symlinks, oversized archives and zip + # bombs before the file is persisted. + if doc_file and is_zip_extension(doc_file.name): + source = doc_file.temporary_file_path() if hasattr(doc_file, "temporary_file_path") else doc_file + try: + validate_safe_zip(source) + except ZipValidationError: + logger.warning("ZIP validation failed for uploaded document.", exc_info=True) + raise forms.ValidationError(_("Invalid or unsafe ZIP archive.")) + finally: + if hasattr(doc_file, "seek"): + try: + doc_file.seek(0) + except (OSError, ValueError): + pass + + if doc_file and os.path.splitext(doc_file.name)[1].lower()[1:] in XML_LIKE_DOCUMENT_EXTENSIONS: + try: + assert_safe_xml(doc_file.read()) + except UnsafeXMLError as err: + logger.warning("Unsafe XML content rejected on document upload: %s", err) + raise forms.ValidationError(_("Uploaded XML contains unsafe content.")) + finally: + if hasattr(doc_file, "seek"): + try: + doc_file.seek(0) + except (OSError, ValueError): + pass + return doc_file diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index 316bdafdfe8..6a51faaae36 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -99,7 +99,9 @@ def create_document_thumbnail(self, object_id): except Document.DoesNotExist: logger.error(f"Document #{object_id} does not exist.") raise - + if not document.is_local: + logger.info(f"Skipping thumbnail generation for remote document: {document.doc_url}") + return image_file = None thumbnail_content = None remove_tmp_file = False diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 278b1f46fb0..e04011b0e15 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -25,6 +25,7 @@ import os import io import json +import zipfile import gisdata from PIL import Image @@ -378,14 +379,81 @@ def test_upload_document_form_size_limit(self): ) self.assertEqual(form.errors, {"doc_file": [expected_error]}) + def test_upload_document_form_rejects_unsafe_zip(self): + """A zip-based document carrying a path-traversal entry must be rejected by the form.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("../../etc/passwd", b"root:x:0:0") + buf.seek(0) + + form_data = { + "title": "Malicious archive", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + } + file_data = {"doc_file": SimpleUploadedFile("evil.zip", buf.read(), "application/zip")} + form = DocumentCreateForm(form_data, file_data) + self.assertFalse(form.is_valid()) + self.assertIn("doc_file", form.errors) + self.assertIn("Invalid or unsafe ZIP archive.", str(form.errors["doc_file"])) + + def test_upload_document_form_accepts_clean_zip(self): + """A well-formed zip document must pass the safety check and validate.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("readme.txt", b"hello") + buf.seek(0) + + form_data = { + "title": "Clean archive", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + } + file_data = {"doc_file": SimpleUploadedFile("clean.zip", buf.read(), "application/zip")} + form = DocumentCreateForm(form_data, file_data) + self.assertTrue(form.is_valid(), msg=form.errors) + + def test_upload_document_form_accepts_valid_xml(self): + iso_xml_content = ( + b'' + b"" + b'123-abc' + b"" + b"" + ) + form_data = { + "title": "ISO Metadata XML", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + } + file_data = {"doc_file": SimpleUploadedFile("metadata.xml", iso_xml_content, "text/plain")} + form = DocumentCreateForm(form_data, file_data) + self.assertTrue(form.is_valid(), msg=form.errors) + def test_document_embed(self): """/documents/1 -> Test accessing the embed view of a document""" d = Document.objects.all().first() d.set_default_permissions() - response = self.client.get(reverse("document_embed", args=(str(d.id),))) + url = reverse("document_embed", args=(str(d.id),)) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + self.assertEqual(self.client.post(url).status_code, 405) + self.assertEqual(self.client.put(url).status_code, 405) + self.assertEqual(self.client.patch(url).status_code, 405) + + def test_resourcebase_embed_is_get_only(self): + d = Document.objects.all().first() + d.set_default_permissions() + + url = reverse("resourcebase_embed", kwargs={"resourcebaseid": d.pk}) + + response = self.client.get(url) self.assertEqual(response.status_code, 200) + self.assertEqual(self.client.post(url).status_code, 405) + self.assertEqual(self.client.put(url).status_code, 405) + self.assertEqual(self.client.patch(url).status_code, 405) + def test_access_document_upload_form(self): """Test the form page is returned correctly via GET request /documents/upload""" @@ -671,6 +739,14 @@ def setUp(self): self.perm_spec = self.__class__.perm_spec self.doc_link_url = self.__class__.doc_link_url + def test_document_link_sets_anti_xss_headers(self): + self.test_doc.set_permissions(self.perm_spec) + self.client.login(username=self.not_admin.username, password="very-secret") + response = self.client.get(self.doc_link_url) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers.get("X-Content-Type-Options"), "nosniff") + self.assertIn("sandbox", response.headers.get("Content-Security-Policy", "")) + def test_document_link_with_permissions(self): self.test_doc.set_permissions(self.perm_spec) # Get link as Anonymous user diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 5b7e7e2121a..92b8eeb6d88 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -28,6 +28,7 @@ from django.template import loader from django.views.generic.edit import CreateView from django.http import HttpResponse, HttpResponseRedirect +from django.views.decorators.http import require_GET from geonode.security.utils import check_add_remote_resource_perm from geonode.base.api.exceptions import geonode_exception_handler @@ -61,6 +62,7 @@ def document_link(request, docid): return response +@require_GET def document_embed(request, docid): document = get_object_or_404(Document, pk=docid) diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index 7fb78d4160e..a6198e47283 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -19,8 +19,6 @@ import json import logging from unittest.mock import MagicMock -from urllib.parse import urljoin - from django.contrib.auth import get_user_model from django.urls import reverse from django.test import override_settings @@ -114,17 +112,6 @@ def test_geoapp_listing_advertised(self): GeoApp.objects.update(advertised=True) - def test_extra_metadata_included_with_param(self): - _app = GeoApp.objects.first() - url = urljoin(f"{reverse('geoapps-list')}/", f"{_app.pk}") - data = {"include[]": "metadata"} - - response = self.client.get(url, format="json", data=data) - self.assertIsNotNone(response.data["geoapp"].get("metadata")) - - response = self.client.get(url, format="json") - self.assertNotIn("metadata", response.data["geoapp"]) - def test_geoapps_crud(self): """ Ensure we can create/update GeoApps. diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index e4094b11848..2bf1eb9fa2f 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -57,6 +57,7 @@ class GeoAppViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicModelViewS GeoAppPermissionsFilter, ] queryset = GeoApp.objects.all().order_by("-created") + search_fields = ["title", "abstract"] serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/geoapps/manager.py b/geonode/geoapps/manager.py index a6186c61177..e02ba87a2d1 100644 --- a/geonode/geoapps/manager.py +++ b/geonode/geoapps/manager.py @@ -36,7 +36,8 @@ def _create_and_update(self, payload, instance=None, notify: bool = True, reques payload = copy.deepcopy(payload) extent = payload.pop("extent", None) - blob = payload.pop("blob", {}) + missing_blob = object() + blob = payload.pop("blob", missing_blob) created = False if not instance: @@ -53,7 +54,9 @@ def _create_and_update(self, payload, instance=None, notify: bool = True, reques logger.exception(f"Error while creating or updating GeoApp instance with exception {e}") raise GeneralGeoAppException("An error occurred while saving the GeoApp.") - payload["blob"] = blob + if blob is not missing_blob: + payload["blob"] = blob + instance = super().update(instance.uuid, instance=instance, vals=payload, notify=notify) if extent or request_user: # Mirrors ResourceBaseSerializer.save() (extent + role defaults); could be moved to the API, diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index 903055016d6..8b884cf7505 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -21,8 +21,10 @@ from django.contrib.auth import get_user_model from geonode.geoapps.models import GeoApp -from geonode.base.models import TopicCategory +from geonode.base.models import TopicCategory, Region, Thesaurus, ThesaurusKeyword +from geonode.groups.models import GroupProfile from geonode.resource.registry import resource_manager_registry, geoapp_manager +from geonode.security.registry import permissions_registry from geonode.tests.base import GeoNodeBaseTestSupport from geonode.metadata.manager import metadata_manager from geonode.base.populate_test_data import all_public, create_models, remove_models @@ -80,13 +82,81 @@ def test_geoapp_category_is_correctly_assigned_in_metadata_upload(self): def test_geoapp_copy(self): self.client.login(username="admin", password="admin") + self.geoapp.blob = {"test_data": {"test": ["test_1", "test_2", "test_3"]}} geoapp_copy = None try: + # owner must be whoever triggers the clone (self.bobby), not self.geoapp's own owner (self.user) geoapp_copy = resource_manager_registry.get_for_instance(self.geoapp).copy( - self.geoapp, defaults=dict(title="Testing GeoApp 2") + self.geoapp, owner=self.bobby, defaults=dict(title="Testing GeoApp 2") ) self.assertIsNotNone(geoapp_copy) self.assertEqual(geoapp_copy.title, "Testing GeoApp 2") + self.assertEqual(geoapp_copy.owner, self.bobby) + self.assertEqual(self.geoapp.blob, geoapp_copy.blob) + finally: + if geoapp_copy: + geoapp_copy.delete() + self.assertIsNotNone(self.geoapp) + + def test_geoapp_embed_is_get_only(self): + self.client.login(username="admin", password="admin") + + url = reverse("geoapp_embed", kwargs={"geoappid": self.geoapp.pk}) + + self.assertEqual(self.client.get(url).status_code, 200) + self.assertEqual(self.client.post(url).status_code, 405) + self.assertEqual(self.client.put(url).status_code, 405) + self.assertEqual(self.client.patch(url).status_code, 405) + + def test_geoapp_copy_carries_over_metadata_and_permissions(self): + """M2M metadata and perm_spec must survive a GeoApp copy too, same as Dataset/Map.""" + self.client.login(username="admin", password="admin") + # update() drops keywords/regions kwargs, set M2M fields directly instead + region = Region.objects.first() + self.geoapp = geoapp_manager.update( + self.geoapp.uuid, instance=self.geoapp, vals={"abstract": "test abstract", "purpose": "test purpose"} + ) + self.geoapp.keywords.add("foo", "bar") + self.geoapp.regions.add(region) + thesaurus = Thesaurus.objects.create(identifier="test_thesaurus_geoapp_copy", title="Test Thesaurus") + tkeyword = ThesaurusKeyword.objects.create(thesaurus=thesaurus, alt_label="test_tkeyword") + self.geoapp.tkeywords.add(tkeyword) + + custom_group, _ = GroupProfile.objects.get_or_create( + slug="geoapp_copy_group", title="geoapp_copy_group", access="private" + ) + custom_perms = { + "users": {self.bobby.username: ["view_resourcebase", "change_resourcebase"]}, + "groups": {custom_group.slug: ["view_resourcebase"]}, + } + geoapp_manager.set_permissions(self.geoapp.uuid, instance=self.geoapp, permissions=custom_perms) + + geoapp_copy = None + try: + geoapp_copy = resource_manager_registry.get_for_instance(self.geoapp).copy( + self.geoapp, owner=self.bobby, defaults=dict(title="Testing GeoApp Metadata Copy") + ) + self.assertIsNotNone(geoapp_copy) + self.assertEqual(geoapp_copy.owner, self.bobby) + self.assertEqual(self.geoapp.abstract, geoapp_copy.abstract) + self.assertEqual(self.geoapp.purpose, geoapp_copy.purpose) + self.assertCountEqual( + [k.name for k in self.geoapp.keywords.all()], [k.name for k in geoapp_copy.keywords.all()] + ) + self.assertCountEqual(list(self.geoapp.regions.all()), list(geoapp_copy.regions.all())) + self.assertCountEqual(list(self.geoapp.tkeywords.all()), list(geoapp_copy.tkeywords.all())) + + source_perms = permissions_registry.get_perms(instance=self.geoapp, include_virtual=False) + copy_perms = permissions_registry.get_perms(instance=geoapp_copy, include_virtual=False) + for perm_key in ("users", "groups"): + source_entries = {profile.pk: set(perms) for profile, perms in source_perms.get(perm_key, {}).items()} + copy_entries = {profile.pk: set(perms) for profile, perms in copy_perms.get(perm_key, {}).items()} + # bobby is also the clone's new owner, who always gets full owner perms on top + # of whatever perm_spec is passed, regardless of what the source spec granted him + source_entries.pop(self.bobby.pk, None) + copy_entries.pop(self.bobby.pk, None) + self.assertEqual(source_entries, copy_entries) + self.assertIn("change_resourcebase_permissions", copy_perms["users"][self.bobby]) finally: if geoapp_copy: geoapp_copy.delete() diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index 3c85a07fd4c..0b3ec29fcae 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -16,7 +16,6 @@ # along with this program. If not, see . # ######################################################################### -import ast import json import logging @@ -25,13 +24,13 @@ from django.utils.translation import gettext_lazy as _ from django.http import HttpResponse, Http404 from django.views.decorators.clickjacking import xframe_options_sameorigin +from django.views.decorators.http import require_GET from django.core.exceptions import PermissionDenied from geonode.groups.models import GroupProfile from geonode.base.auth import get_or_create_token from geonode.security.views import _perms_info_json from geonode.geoapps.models import GeoApp -from geonode.resource.registry import resource_manager_registry from geonode.utils import resolve_object from geonode.security.registry import permissions_registry @@ -52,6 +51,7 @@ def _resolve_geoapp(request, id, permission="base.change_resourcebase", msg=_PER @xframe_options_sameorigin +@require_GET def geoapp_embed(request, geoappid, template="apps/app_embed.html"): """ The view that returns the app composer opened to @@ -79,15 +79,6 @@ def geoapp_embed(request, geoappid, template="apps/app_embed.html"): group = None r = geoapp_obj - if request.method in ("POST", "PATCH", "PUT"): - resolved_resource_manager = resource_manager_registry.get_for_instance(geoapp_obj) - r = resolved_resource_manager.update(geoapp_obj.uuid, instance=geoapp_obj, notify=True) - - resolved_resource_manager.set_permissions( - geoapp_obj.uuid, instance=geoapp_obj, permissions=ast.literal_eval(permissions_json) - ) - - resolved_resource_manager.set_thumbnail(geoapp_obj.uuid, instance=geoapp_obj, overwrite=False) access_token = None if request and request.user: diff --git a/geonode/geoserver/gwc.py b/geonode/geoserver/gwc.py new file mode 100644 index 00000000000..9e54f967203 --- /dev/null +++ b/geonode/geoserver/gwc.py @@ -0,0 +1,120 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import requests + +from xml.sax.saxutils import escape + +from geoserver.catalog import FailedRequestError +from requests.auth import HTTPBasicAuth + +from geonode.geoserver.helpers import ogc_server_settings +from django.conf import settings + +logger = logging.getLogger(__name__) + + +class GWCClient: + """ + A GeoWebCache REST client for interacting with the GWC API. + """ + + def __init__(self, **kwargs) -> None: + self.base_url = f"{ogc_server_settings.LOCATION}gwc/rest/" + self.debug = kwargs.get("debug", False) + + def __log_response(self, r): + if self.debug: + logger.debug(f'{r.request.method} response: code:{r.status_code} --> "{r.text}"') + else: + logger.debug(f"{r.request.method} response: code:{r.status_code} --> {len(r.text)} bytes") + + def _get(self, urlpath): + _user, _password = ogc_server_settings.credentials + + url = f"{self.base_url}{urlpath}" + r = requests.get(url=url, auth=HTTPBasicAuth(_user, _password), timeout=30) + self.__log_response(r) + return r + + def _post(self, urlpath, data): + _user, _password = ogc_server_settings.credentials + + url = f"{self.base_url}{urlpath}" + r = requests.post(url=url, data=data, auth=HTTPBasicAuth(_user, _password), timeout=30) + self.__log_response(r) + return r + + def _put(self, urlpath, data): + _user, _password = ogc_server_settings.credentials + + url = f"{self.base_url}{urlpath}" + r = requests.put( + url=url, data=data, headers={"Content-Type": "text/xml"}, auth=HTTPBasicAuth(_user, _password), timeout=30 + ) + self.__log_response(r) + return r + + @staticmethod + def _validate_layer_name(layer_name: str, workspace: str | None = None) -> str: + if ":" not in layer_name: + workspace = workspace or getattr(settings, "DEFAULT_WORKSPACE", "geonode") + logger.info( + "Workspace not provided for layer '%s'. Using default workspace '%s'.", + layer_name, + workspace, + ) + + layer_name = f"{workspace}:{layer_name}" + return layer_name + + def get_layer(self, layer_name: str, workspace: str | None = None) -> requests.Response: + """ + Get GWC layer information. Returns the raw response from the GWC API. + """ + layer_name = self._validate_layer_name(layer_name, workspace) + return self._get(f"layers/{layer_name}.xml") + + def set_layer(self, layer_name: str, workspace: str | None = None, data=None) -> requests.Response: + layer_name = self._validate_layer_name(layer_name, workspace) + return self._put(f"layers/{layer_name}.xml", data=data) + + def truncate_layer(self, layer_name: str, workspace: str | None = None) -> None: + """ + Truncate all cached tiles for a GWC layer. + """ + layer_name = self._validate_layer_name(layer_name, workspace) + body = f"{escape(layer_name)}" + r = self._post("masstruncate", body) + if r.status_code != 200: + raise FailedRequestError(f'Error truncating GWC layer: {r.status_code}: "{r.text}"') + + logger.info("Successfully truncated GWC cache for layer '%s'.", layer_name) + + def truncate_all(self) -> None: + """ + Clear the entire GWC cache. + """ + r = self._post("masstruncate", "") + + if r.status_code != 200: + raise FailedRequestError(f'Error truncating all GWC layers: {r.status_code}: "{r.text}"') + + logger.info("Successfully truncated the whole GWC cache.") diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index a165d90bbc7..1cd5d568a64 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -31,7 +31,7 @@ from itertools import cycle from collections import defaultdict from os.path import basename, splitext, isfile -from urllib.parse import urlparse, urlencode, urlsplit, urljoin +from urllib.parse import urlparse, urlunparse, urlencode, urlsplit, urljoin, parse_qsl from bs4 import BeautifulSoup import xml.etree.ElementTree as ET @@ -76,6 +76,7 @@ is_monochromatic_image, set_resource_default_links, normalize_bbox_to_float_list, + safe_request_url, ) from .geofence import GeoFenceClient, GeoFenceUtils @@ -111,7 +112,8 @@ def _sync_geoserver_keywords_to_instance(instance, keywords): if not keywords: return try: - KeywordHandler(instance=instance, keywords=list(keywords)).set_keywords() + merged_keywords = set(keywords) | set(instance.keyword_list()) + KeywordHandler(instance=instance, keywords=list(merged_keywords)).set_keywords() except Exception: logger.exception(f"Error while importing keywords from GeoServer for dataset {instance.name}") @@ -605,6 +607,7 @@ def gs_slurp( filter=None, skip_unadvertised=False, skip_geonode_registered=False, + skip_keywords=False, remove_deleted=False, permissions=None, execute_signals=False, @@ -763,6 +766,10 @@ def gs_slurp( # recalculate the layer statistics set_attributes_from_geoserver(layer, overwrite=True) + if not skip_keywords: + # import the keywords from GeoServer, merging them with the existing ones + _sync_geoserver_keywords_to_instance(layer, resource.keywords) + # in some cases we need to explicitily save the resource to execute the signals # (for sure when running updatelayers) resolved_resource_manager.update(layer.uuid, instance=layer, notify=execute_signals) @@ -997,6 +1004,10 @@ def set_attributes_from_geoserver(layer, overwrite=False): then store in GeoNode database using Attribute model """ attribute_map = [] + if layer.subtype == "remote" and not layer.remote_service: + # avoid falling through to the local-GeoServer branches, which would wipe existing attributes + logger.debug(f"Dataset '{layer.alternate or layer.typename}' is remote but has no remote_service yet") + return if getattr(layer, "remote_service") and layer.remote_service: server_url = layer.remote_service.service_url if layer.remote_service.operations.get("GetCapabilities", None) and layer.remote_service.operations.get( @@ -1008,20 +1019,40 @@ def set_attributes_from_geoserver(layer, overwrite=False): break else: server_url = ogc_server_settings.LOCATION - if layer.subtype in ["tileStore", "remote"] and layer.remote_service.ptype == "gxp_arcrestsource": - logger.info(f"Getting info for {layer.subtype} '{layer.alternate or layer.typename}'") - dft_url = f"{server_url}{(layer.alternate or layer.typename)}?f=json" + if layer.subtype == "remote" and layer.remote_service and layer.remote_service.ptype == "gxp_arcrestsource": + resource_identifier = layer.alternate or layer.typename + logger.info(f"Getting info for {layer.subtype} '{resource_identifier}'") + # `resource_identifier` is GeoNode's own "workspace:name" form (e.g. "remoteWorkspace:0"), + # not a real ArcGIS REST path: the actual layer id is whatever follows the last ':'. + arcgis_layer_id = resource_identifier.rsplit(":", 1)[-1] + # server_url may already carry a query string (e.g. an auth token via extra_queryparams): + # append the layer id to the path and merge f=json into the existing query, not the string. + parsed_server_url = urlparse(server_url) + query = dict(parse_qsl(parsed_server_url.query)) + query["f"] = "json" + dft_url = urlunparse( + parsed_server_url._replace( + path=f"{parsed_server_url.path.rstrip('/')}/{arcgis_layer_id}", + query=urlencode(query), + ) + ) try: - # The code below will fail if http_client cannot be imported - req, body = http_client.get(dft_url, user=_user) - body = json.loads(body) - attribute_map = [ - [n["name"], _esri_types[n["type"]]] for n in body["fields"] if n.get("name") and n.get("type") - ] + # Not http_client(user=...): that would send our own GeoServer OAuth token to this third party. + remote_auth = None + if layer.remote_service.needs_authentication: + from geonode.security.auth_registry import auth_handler_registry + + remote_auth = auth_handler_registry.build(layer.remote_service.auth_config).get_request_auth() + response = safe_request_url("GET", dft_url, auth=remote_auth, timeout=10) + response.raise_for_status() + body = response.json() + attribute_map = [] + for n in body["fields"]: + esri_type = _esri_types.get(n.get("type")) + if n.get("name") and esri_type: + attribute_map.append([n["name"], esri_type]) except Exception: - logger.warning( - f"Error while retrieving info for {layer.subtype} '{layer.alternate or layer.typename}'", exc_info=True - ) + logger.warning(f"Error while retrieving info for {layer.subtype} '{resource_identifier}'", exc_info=True) attribute_map = [] elif layer.can_have_wps_links: typename = layer.alternate if layer.alternate else layer.typename @@ -1932,6 +1963,7 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs): """ updatebbox = kwargs.get("updatebbox", True) updatemetadata = kwargs.get("updatemetadata", True) + importgskeywords = kwargs.get("importgskeywords", False) instance = None try: @@ -1979,7 +2011,8 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs): instance = instance.fixup_store_type(["alternate", "store", "subtype"], values) if updatemetadata: - _sync_geoserver_keywords_to_instance(instance, gs_resource.keywords) + if importgskeywords: + _sync_geoserver_keywords_to_instance(instance, gs_resource.keywords) # Get metadata links metadata_links = [] @@ -2058,7 +2091,7 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs): "alternate": instance.alternate, } - if updatebbox and is_monochromatic_image(instance.thumbnail_url): + if updatebbox and is_monochromatic_image(instance.thumbnail_url, image_path=instance.thumbnail_path): to_update["thumbnail_url"] = None # Save all the modified information in the instance without triggering signals. diff --git a/geonode/geoserver/management/commands/gwc.py b/geonode/geoserver/management/commands/gwc.py new file mode 100644 index 00000000000..1cd1698451e --- /dev/null +++ b/geonode/geoserver/management/commands/gwc.py @@ -0,0 +1,73 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand, CommandError + +from geonode.base.management.command_utils import setup_logger +from geonode.geoserver.management.commands.gwc_subcommands import truncate +from geonode.geoserver.management.commands.gwc_subcommands.create import CreateTileLayers + +logger = setup_logger() + +COMMAND_TRUNCATE = "truncate" +COMMAND_CREATE = "create" +COMMANDS = [COMMAND_CREATE, COMMAND_TRUNCATE] + + +class Command(BaseCommand): + help = f"Handles GWC commands {COMMANDS}" + + def _add_common_arguments(self, parser): + parser.add_argument( + '-d', + '--dry-run', + dest="dry-run", + action='store_true', + help="Do not actually perform any change on GWC") + + parser.add_argument( + '--debug', + dest="debug", + action='store_true', + help="Show debug logging") + + def add_arguments(self, parser): + + subparsers = parser.add_subparsers(dest="subcommand", required=True) + + parser_truncate = subparsers.add_parser(COMMAND_TRUNCATE, help="Truncate tile layers") + truncate.add_arguments(parser_truncate) + self._add_common_arguments(parser_truncate) + + parser_create = subparsers.add_parser(COMMAND_CREATE, help="Create tile layers") + CreateTileLayers().add_arguments(parser_create) + self._add_common_arguments(parser_create) + + def handle(self, *args, **options): + subcommand = options["subcommand"] + logger.info(f"Executing GWC subcommand '{subcommand}'...") + + if subcommand == COMMAND_TRUNCATE: + truncate.handle(options) + elif subcommand == COMMAND_CREATE: + CreateTileLayers().handle(**options) + else: + raise CommandError(f"Unknown subcommand: {subcommand}") + + logger.info(f"GWC subcommand {subcommand} completed.") diff --git a/geonode/geoserver/management/commands/gwc_subcommands/__init__.py b/geonode/geoserver/management/commands/gwc_subcommands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/geoserver/management/commands/gwc_subcommands/create.py b/geonode/geoserver/management/commands/gwc_subcommands/create.py new file mode 100644 index 00000000000..378d80aa6bc --- /dev/null +++ b/geonode/geoserver/management/commands/gwc_subcommands/create.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import os +import xml.etree.ElementTree as ET + +from django.core.management import CommandError + +from geonode.base.management.command_utils import setup_logger +from geonode.geoserver.gwc import GWCClient +from geonode.layers.models import Dataset + + +logger = setup_logger() + + +class CreateTileLayers: + help = "Create TileLayers in GWC" + + def add_arguments(self, parser): + parser.add_argument( + '-f', + '--force', + dest="force", + action='store_true', + help="Force tile layer re-creation also if it already exists in GWC") + + parser.add_argument( + '-l', + '--layer', + dest="layers", + action='append', + help="Only process specified layers ") + + parser.add_argument( + "--all", + dest="create_all", + action="store_true", + help="Create caches for all layers", + ) + + parser.add_argument( + "-t", + "--template", + dest="xmltemplatefilepath", + help="Use the specified XML template file path", + ) + + def handle(self, **options): + dry_run = options.get('dry-run') + debug = options.get('debug') + setup_logger(level=logging.DEBUG if debug else logging.INFO) + + force = options.get('force') + + requested_layers = options.get('layers', []) + create_all = options.get('create_all') + + if not requested_layers and not create_all: + raise CommandError("'create' command requires either the -l/--layer parameter(s) or the --all flag.") + + if requested_layers and create_all: + raise CommandError("Cannot use both -l/--layer and --all at the same time.") + + xmltemplate = options.get('xmltemplatefilepath') + + if debug: + logger.debug(f"FORCE is {force}") + logger.debug(f"DRY-RUN is {dry_run}") + logger.debug(f"LAYERS is {requested_layers}") + + self.run(requested_layers=requested_layers, create_all=create_all, force=force, xmltemplate=xmltemplate, dry_run=dry_run, debug=debug) + + def run(self, requested_layers=None, create_all=False, force=False, xmltemplate=None, dry_run=False, debug=False): + if create_all: + datasets_fields = Dataset.objects.values_list('typename', 'alternate') + layer_names = [typename or alternate for typename, alternate in datasets_fields] + else: + layer_names = requested_layers + tot = len(layer_names) + logger.info(f"Total layers to be processed: {tot}") + + template = self.load_xml_template(xmltemplate, debug=debug) + gwc = GWCClient(debug=debug) + + i = cnt_old = cnt_new = cnt_bad = cnt_force = 0 + + for layername in layer_names: + i += 1 + logger.info(f"- {i}/{tot} Processing layer: {layername}") + + r = gwc.get_layer(layername) + if r.status_code == 200: + if force: + logger.info(" - Forcing layer configuration in GWC") + cnt_force += 1 + else: + logger.info(f" - Layer already configured in GWC (code {r.status_code})") + cnt_old += 1 + continue + try: + logger.info(" - Configuring...") + data = self.generate_xml(template, layername) + if not dry_run: + response = gwc.set_layer(layername, data=data) + + if dry_run or response.status_code == 200: + logger.info(f" - Done {layername}") + cnt_new += 1 + else: + logger.warning(f"Layer {layername} couldn't be configured: code {response.status_code}") + cnt_bad += 1 + + except Exception as e: + logger.warning(f"Error processing {layername}: {e}") + cnt_bad += 1 + + logger.info("Work completed") + logger.info(f"- TileLayers configured: {cnt_new}" + (f" (forced {cnt_force})" if cnt_force else "")) + logger.info(f"- TileLayers in error : {cnt_bad}") + logger.info(f"- TileLayers untouched : {cnt_old}") + + def load_xml_template(self, xmltemplate=None, debug=False): + """Load the XML template""" + if xmltemplate: + logger.info(f"Using provided xml template at {xmltemplate}") + xml_path = os.path.abspath(xmltemplate) + else: + current_dir = os.path.dirname(os.path.abspath(__file__)) + xml_path = os.path.join(current_dir, "create_template.xml") + + if not os.path.isfile(xml_path): + raise CommandError(f"XML template could not be found at {xml_path}") + + try: + tree = ET.parse(xml_path) + except ET.ParseError as e: + if debug: + logger.exception(f"Error parsing XML template {xml_path}") + raise CommandError(f"Error parsing XML template {xml_path}: {e}") + + root = tree.getroot() + return root + + def generate_xml(self, root, layer_name): + name_node = root.find('name') + + if name_node is None: + raise ValueError("name node not found in XML template") + + name_node.text = layer_name + return ET.tostring(root, encoding="utf-8", method="xml") diff --git a/geonode/geoserver/management/commands/gwc_subcommands/create_template.xml b/geonode/geoserver/management/commands/gwc_subcommands/create_template.xml new file mode 100644 index 00000000000..a907561b566 --- /dev/null +++ b/geonode/geoserver/management/commands/gwc_subcommands/create_template.xml @@ -0,0 +1,41 @@ + + + true + true + {} + + 2 + 2 + + + application/json;type=utfgrid + image/gif + image/jpeg + image/png + image/png8 + image/vnd.jpeg-png + image/vnd.jpeg-png8 + + + + EPSG:3857 + + + EPSG:3857x2 + + + EPSG:4326 + + + EPSG:4326x2 + + + EPSG:900913 + + + EPSG:900913x2 + + + true + 0 + diff --git a/geonode/geoserver/management/commands/gwc_subcommands/truncate.py b/geonode/geoserver/management/commands/gwc_subcommands/truncate.py new file mode 100644 index 00000000000..2f0e487c400 --- /dev/null +++ b/geonode/geoserver/management/commands/gwc_subcommands/truncate.py @@ -0,0 +1,100 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from geonode.base.management.command_utils import setup_logger +from geonode.geoserver.gwc import GWCClient + +from django.core.management.base import CommandError + +logger = setup_logger() + + +def add_arguments(parser): + parser.add_argument( + "-l", + "--layer", + dest="layers", + action="append", + help="Name of the layer(s) to truncate. Can be repeated.", + ) + + parser.add_argument( + "--all", + dest="truncate_all", + action="store_true", + help="Truncate all caches in GWC", + ) + + +def handle(options: dict) -> None: + dry_run = options.get('dry-run', False) + debug = options.get('debug', False) + setup_logger(level=logging.DEBUG if debug else logging.INFO) + + layers = options.get("layers", []) + truncate_all = options.get("truncate_all") + + if not layers and not truncate_all: + raise CommandError("'truncate' command requires either the -l/--layer parameter(s) or the --all flag.") + + if layers and truncate_all: + raise CommandError("Cannot use both -l/--layer and --all at the same time.") + + logger.info( + "Truncate command received. layers=%s, truncate_all=%s", + layers, + truncate_all, + ) + + if truncate_all: + truncate_all_layers(dryrun=dry_run, debug=debug) + else: + logger.info(f"Truncating {len(layers)} layer{'s' if len(layers) > 1 else ''}") + truncate_layers(layers, dryrun=dry_run, debug=debug) + + +def truncate_layers(layer_names, dryrun=False, debug=False): + """Truncates one or more specified layers in GWC.""" + logger.info(f"Truncating layers in GWC: {', '.join(layer_names)}") + client = GWCClient(debug=debug) + failed_layers = [] + for layer_name in layer_names: + try: + logger.info(f"Truncating layer: {layer_name}") + if not dryrun: + client.truncate_layer(layer_name) + except Exception as e: + logger.error(f"Error invalidating cache for layer {layer_name}: {e}") + failed_layers.append(layer_name) + if failed_layers: + raise CommandError(f"Failed to truncate the following layers: {', '.join(failed_layers)}") + + +def truncate_all_layers(dryrun=False, debug=False): + """Truncates all layers in GWC.""" + logger.info("Truncating ALL layers in GWC") + client = GWCClient(debug=debug) + try: + if not dryrun: + client.truncate_all() + logger.info("Successfully truncated all layers in GWC.") + except Exception as e: + logger.error(f"Error executing truncateAll on GeoWebCache: {e}") + raise CommandError(f"Error executing truncateAll on GeoWebCache: {e}") diff --git a/geonode/geoserver/management/commands/sync_geonode_datasets.py b/geonode/geoserver/management/commands/sync_geonode_datasets.py index b9f16561de8..c75b2d23ef4 100644 --- a/geonode/geoserver/management/commands/sync_geonode_datasets.py +++ b/geonode/geoserver/management/commands/sync_geonode_datasets.py @@ -75,7 +75,7 @@ def sync_geonode_datasets( sync_instance_with_geoserver(layer.id, updatemetadata=False, updatebbox=True) if updatemetadata: logger.info("Updating metadata...") - sync_instance_with_geoserver(layer.id, updatemetadata=True, updatebbox=False) + sync_instance_with_geoserver(layer.id, updatemetadata=True, updatebbox=False, importgskeywords=True) if removeduplicates: # remove duplicates logger.info("Removing duplicate links...") diff --git a/geonode/geoserver/management/commands/updatelayers.py b/geonode/geoserver/management/commands/updatelayers.py index a00d03bb4ec..ee38e4de1ce 100644 --- a/geonode/geoserver/management/commands/updatelayers.py +++ b/geonode/geoserver/management/commands/updatelayers.py @@ -48,6 +48,12 @@ def add_arguments(self, parser): dest='skip_geonode_registered', default=False, help='Just processing GeoServer layers still not registered in GeoNode.'), + parser.add_argument( + '--skip-keywords', + action='store_true', + dest='skip_keywords', + default=False, + help='Do not import GeoServer keywords into the GeoNode datasets.'), parser.add_argument( '--remove-deleted', action='store_true', @@ -89,6 +95,7 @@ def handle(self, **options): ignore_errors = options.get('ignore_errors') skip_unadvertised = options.get('skip_unadvertised') skip_geonode_registered = options.get('skip_geonode_registered') + skip_keywords = options.get('skip_keywords') remove_deleted = options.get('remove_deleted') verbosity = int(options.get('verbosity')) user = options.get('user') @@ -116,6 +123,7 @@ def handle(self, **options): filter=filter, skip_unadvertised=skip_unadvertised, skip_geonode_registered=skip_geonode_registered, + skip_keywords=skip_keywords, remove_deleted=remove_deleted, permissions=permissions, execute_signals=True) diff --git a/geonode/geoserver/signals.py b/geonode/geoserver/signals.py index 0f3a84768bc..a4ac3ae5c73 100644 --- a/geonode/geoserver/signals.py +++ b/geonode/geoserver/signals.py @@ -144,7 +144,9 @@ def geoserver_set_thumbnail(instance, **kwargs): and "thumbnail_url" in kwargs["update_fields"] ): _recreate_thumbnail = True - if not instance.thumbnail_url or is_monochromatic_image(instance.thumbnail_url): + if not instance.thumbnail_url or is_monochromatic_image( + instance.thumbnail_url, image_path=instance.thumbnail_path + ): _recreate_thumbnail = True if _recreate_thumbnail: geoserver_create_thumbnail.apply_async( diff --git a/geonode/geoserver/tests/test_gwc.py b/geonode/geoserver/tests/test_gwc.py new file mode 100644 index 00000000000..3e37e1e6636 --- /dev/null +++ b/geonode/geoserver/tests/test_gwc.py @@ -0,0 +1,157 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from unittest.mock import patch, MagicMock +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase + +from geoserver.catalog import FailedRequestError +from geonode.geoserver.gwc import GWCClient + + +class GWCClientTest(TestCase): + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_layer_calls_api(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + client = GWCClient() + client.truncate_layer("test_layer1") + + self.assertEqual(mock_post.call_count, 1) + + call_kwargs = mock_post.call_args[1] + self.assertEqual( + call_kwargs["data"], + "geonode:test_layer1", + ) + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_layer_preserves_workspace(self, mock_post): + """Layer already qualified should not get default workspace prepended.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + client = GWCClient() + client.truncate_layer("workspace:test_layer") + + call_kwargs = mock_post.call_args[1] + self.assertEqual( + call_kwargs["data"], + "workspace:test_layer", + ) + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_layer_escapes_xml(self, mock_post): + """Ensure XML injection is prevented via escaping.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + client = GWCClient() + client.truncate_layer("layer&<>name") + + call_kwargs = mock_post.call_args[1]["data"] + + self.assertIn("layer&<>name", call_kwargs) + self.assertNotIn("", call_kwargs) + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_layer_raises_on_error(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + mock_post.return_value = mock_response + + client = GWCClient() + + with self.assertRaises(FailedRequestError): + client.truncate_layer("test_layer") + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_all_calls_api(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + client = GWCClient() + client.truncate_all() + + self.assertEqual(mock_post.call_count, 1) + self.assertEqual( + mock_post.call_args[1]["data"], + "", + ) + + @patch("geonode.geoserver.gwc.requests.post") + def test_truncate_all_raises_on_error(self, mock_post): + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + mock_post.return_value = mock_response + + client = GWCClient() + + with self.assertRaises(FailedRequestError): + client.truncate_all() + + +class GwcManagementCommandTest(TestCase): + + @patch("geonode.geoserver.management.commands.gwc_subcommands.truncate.GWCClient") + def test_management_command_layers(self, MockGWCClient): + """Ensure multiple layers trigger truncate_layer calls.""" + call_command("gwc", "truncate", "-l", "layer1", "-l", "layer2") + + instance = MockGWCClient.return_value + + self.assertEqual(instance.truncate_layer.call_count, 2) + instance.truncate_layer.assert_any_call("layer1") + instance.truncate_layer.assert_any_call("layer2") + + @patch("geonode.geoserver.management.commands.gwc_subcommands.truncate.GWCClient") + def test_management_command_all(self, MockGWCClient): + """Ensure --all triggers truncate_all.""" + call_command("gwc", "truncate", "--all") + + MockGWCClient.return_value.truncate_all.assert_called_once() + + def test_management_command_error_missing_args(self): + """Command must fail when no arguments are provided.""" + with self.assertRaises(CommandError) as context: + call_command("gwc", "truncate") + + self.assertIn( + "requires either the -l/--layer parameter(s) or the --all flag", + str(context.exception), + ) + + def test_management_command_error_both_args(self): + """Command must fail when both layer and --all are used.""" + with self.assertRaises(CommandError) as context: + call_command("gwc", "truncate", "-l", "layer1", "--all") + + self.assertIn( + "Cannot use both -l/--layer and --all at the same time", + str(context.exception), + ) diff --git a/geonode/geoserver/tests/test_helpers.py b/geonode/geoserver/tests/test_helpers.py index 1c32898d6b1..54ad2f105ad 100644 --- a/geonode/geoserver/tests/test_helpers.py +++ b/geonode/geoserver/tests/test_helpers.py @@ -19,6 +19,7 @@ import re import time import logging +import requests from urllib.parse import urljoin @@ -43,8 +44,12 @@ get_dataset_capabilities_url, get_layer_ows_url, get_time_info, + set_attributes_from_geoserver, + _sync_geoserver_keywords_to_instance, ) from geonode.geoserver.ows import _wcs_link, _wfs_link, _wms_link +from geonode.services.models import Service +from geonode.services.enumerations import REST_MAP, INDEXED from unittest.mock import patch, Mock @@ -207,6 +212,97 @@ def test_remoteStore_should_return_remote(self): el = get_dataset_storetype("remoteStore") self.assertEqual("remote", el) + def _create_arcgis_dataset(self): + service = Service.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + type=REST_MAP, + method=INDEXED, + base_url="https://sampleserver6.arcgisonline.com/arcgis/rest/services/USA/MapServer", + name="arcgis-test-service", + title="ArcGIS test service", + ) + return Dataset.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + name="usa_states", + store="arcgis-test-service", + subtype="remote", + workspace="remoteWorkspace", + typename="remoteWorkspace:0", + alternate="remoteWorkspace:0", + remote_service=service, + ) + + @patch("geonode.geoserver.helpers.safe_request_url") + def test_set_attributes_from_geoserver_arcgis_remote_dataset(self, mock_safe_request_url): + dataset = self._create_arcgis_dataset() + mock_response = Mock() + mock_response.json.return_value = { + "fields": [ + {"name": "STATE_NAME", "type": "esriFieldTypeString"}, + {"name": "POPULATION", "type": "esriFieldTypeDouble"}, + {"name": "SOME_FIELD", "type": "esriFieldTypeNotSupported"}, + ] + } + mock_safe_request_url.return_value = mock_response + + set_attributes_from_geoserver(dataset) + + method, called_url = mock_safe_request_url.call_args[0] + self.assertEqual(method, "GET") + self.assertTrue( + called_url.startswith("https://sampleserver6.arcgisonline.com/arcgis/rest/services/USA/MapServer/0") + ) + self.assertIn("f=json", called_url) + attributes = set(dataset.attribute_set.values_list("attribute", "attribute_type")) + self.assertIn(("STATE_NAME", "xsd:string"), attributes) + self.assertIn(("POPULATION", "xsd:double"), attributes) + self.assertFalse(dataset.attribute_set.filter(attribute="SOME_FIELD").exists()) + + @patch("geonode.geoserver.helpers.safe_request_url") + def test_set_attributes_from_geoserver_arcgis_preserves_existing_query_string(self, mock_safe_request_url): + dataset = self._create_arcgis_dataset() + dataset.remote_service.extra_queryparams = "token=abc123" + dataset.remote_service.save() + mock_response = Mock() + mock_response.json.return_value = {"fields": [{"name": "STATE_NAME", "type": "esriFieldTypeString"}]} + mock_safe_request_url.return_value = mock_response + + set_attributes_from_geoserver(dataset) + + called_url = mock_safe_request_url.call_args[0][1] + self.assertIn("token=abc123", called_url) + self.assertIn("f=json", called_url) + + @patch("geonode.geoserver.helpers.safe_request_url") + def test_set_attributes_from_geoserver_arcgis_http_error(self, mock_safe_request_url): + dataset = self._create_arcgis_dataset() + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("500 error") + mock_safe_request_url.return_value = mock_response + + set_attributes_from_geoserver(dataset) + + self.assertFalse(dataset.attribute_set.exists()) + + def test_set_attributes_from_geoserver_arcgis_no_remote_service_yet(self): + dataset = Dataset.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + name="usa_states_pending", + store="arcgis-test-service", + subtype="remote", + workspace="remoteWorkspace", + typename="remoteWorkspace:1", + alternate="remoteWorkspace:1", + ) + Attribute.objects.create(dataset=dataset, attribute="STATE_NAME", attribute_type="xsd:string") + + set_attributes_from_geoserver(dataset, overwrite=True) + + self.assertTrue(dataset.attribute_set.filter(attribute="STATE_NAME").exists()) + @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_geoserver_proxy_strip_paths(self): response = self.client.get( @@ -360,3 +456,29 @@ def test_get_time_info_no_layer(self, mock_gs_catalog): result = get_time_info(mock_layer) self.assertIsNone(result) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_sync_geoserver_keywords_merges_and_does_not_replace(self): + dataset = Dataset.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + name="keywords_merge_test", + store="httpfooremoteservce", + subtype="remote", + alternate="geonode:keywords_merge_test", + ) + dataset.keywords.add("keyword_from_geonode", "shared_keyword") + + _sync_geoserver_keywords_to_instance(dataset, ["keyword_from_geoserver", "shared_keyword"]) + + self.assertSetEqual( + {"keyword_from_geonode", "keyword_from_geoserver", "shared_keyword"}, + set(dataset.keyword_list()), + ) + + # an empty keyword list from GeoServer must leave the existing keywords untouched + _sync_geoserver_keywords_to_instance(dataset, []) + self.assertSetEqual( + {"keyword_from_geonode", "keyword_from_geoserver", "shared_keyword"}, + set(dataset.keyword_list()), + ) diff --git a/geonode/groups/forms.py b/geonode/groups/forms.py index 376dd4d464e..bd3737aa7e9 100644 --- a/geonode/groups/forms.py +++ b/geonode/groups/forms.py @@ -25,6 +25,7 @@ from geonode.groups.models import GroupProfile from geonode.base.widgets import TaggitSelect2Custom +from geonode.people.utils import contains_disallowed_template_tokens class GroupForm(forms.ModelForm): @@ -46,6 +47,12 @@ def clean_title(self): def clean(self): cleaned_data = self.cleaned_data + for field_name, value in list(cleaned_data.items()): + if contains_disallowed_template_tokens(value): + raise forms.ValidationError( + _("Field %(field_name)s contains characters that are not allowed."), + params={"field_name": field_name}, + ) name = cleaned_data.get("title") if not name or GroupProfile.objects.filter(title__iexact=self.cleaned_data["title"]).exists(): @@ -71,6 +78,16 @@ def clean_name(self): raise forms.ValidationError(_("A group already exists with that name.")) return self.cleaned_data["title"] + def clean(self): + cleaned_data = self.cleaned_data + for field_name, value in list(cleaned_data.items()): + if contains_disallowed_template_tokens(value): + raise forms.ValidationError( + _("Field %(field_name)s contains characters that are not allowed."), + params={"field_name": field_name}, + ) + return cleaned_data + class Meta: model = GroupProfile exclude = ["group"] diff --git a/geonode/groups/migrations/0036_fix_groupprofile_access_default.py b/geonode/groups/migrations/0036_fix_groupprofile_access_default.py new file mode 100644 index 00000000000..bb922454f11 --- /dev/null +++ b/geonode/groups/migrations/0036_fix_groupprofile_access_default.py @@ -0,0 +1,38 @@ +from django.db import migrations, models + +# The old default, with a stray apostrophe that matches none of the field's own +# GROUP_CHOICES. Every GroupProfile created without an explicit access — which is +# what geonode_ldap's group mirroring does — was stored with this value. +BAD_DEFAULT = "public'" + + +def repair_access(apps, schema_editor): + GroupProfile = apps.get_model("groups", "GroupProfile") + GroupProfile.objects.filter(access=BAD_DEFAULT).update(access="public") + + +class Migration(migrations.Migration): + dependencies = [ + ("groups", "0035_remove_modeltranslation"), + ] + + operations = [ + migrations.AlterField( + model_name="groupprofile", + name="access", + field=models.CharField( + choices=[ + ("public", "Public"), + ("public-invite", "Public (invite-only)"), + ("private", "Private"), + ], + default="public", + help_text="Public: Any registered user can view and join a public group.
Public (invite-only):Any registered user can view the group. Only invited users can join.
Private: Registered users cannot see any details about the group, including membership. Only invited users can join.", + max_length=15, + verbose_name="Access", + ), + ), + # Reversing would mean writing the invalid value back, so this only goes + # forward; the AlterField above is reversible on its own. + migrations.RunPython(repair_access, migrations.RunPython.noop), + ] diff --git a/geonode/groups/models.py b/geonode/groups/models.py index a4cddca47db..b70511ef0e4 100644 --- a/geonode/groups/models.py +++ b/geonode/groups/models.py @@ -91,7 +91,7 @@ class GroupProfile(models.Model): email = models.EmailField(_("Email"), null=True, blank=True, help_text=email_help_text) keywords = TaggableManager(_("Keywords"), help_text=_("A space or comma-separated list of keywords"), blank=True) access = models.CharField( - _("Access"), max_length=15, default="public'", choices=GROUP_CHOICES, help_text=access_help_text + _("Access"), max_length=15, default="public", choices=GROUP_CHOICES, help_text=access_help_text ) categories = models.ManyToManyField(GroupCategory, verbose_name=_("Categories"), blank=True, related_name="groups") created = models.DateTimeField(auto_now_add=True, null=True, blank=True) @@ -276,14 +276,16 @@ class GroupMember(models.Model): joined = models.DateTimeField(default=now) def save(self, *args, **kwargs): - # add django.contrib.auth.group to user - self.user.groups.add(self.group.group) super().save(*args, **kwargs) + # add django.contrib.auth.group to user + if not self.user.groups.filter(id=self.group.group.id).exists(): + self.user.groups.add(self.group.group) self._handle_perms(role=self.role) def delete(self, *args, **kwargs): - self.user.groups.remove(self.group.group) super().delete(*args, **kwargs) + if self.user.groups.filter(id=self.group.group.id).exists(): + self.user.groups.remove(self.group.group) self._handle_perms() def promote(self, *args, **kwargs): diff --git a/geonode/groups/templates/groups/group_detail.html b/geonode/groups/templates/groups/group_detail.html index ca14b7d1bbd..884af946702 100644 --- a/geonode/groups/templates/groups/group_detail.html +++ b/geonode/groups/templates/groups/group_detail.html @@ -16,7 +16,7 @@ {% block body_class %}groups{% endblock %} {% block body_outer %} -
+

{{ object.title|default:object.slug }}

{% trans "Last Modified" %}: {{ object.last_modified }} @@ -25,14 +25,14 @@

{{ object.title|default:object.slug }}

-