Update database sql files

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       mariadb:
-        image: mariadb:10.8
+        image: mariadb:10.6
         env:
           MYSQL_USER: test
           MYSQL_PASSWORD: test
@@ -28,7 +28,7 @@ jobs:
       - uses: shogo82148/actions-setup-mysql@v1
         with:
           distribution: 'mariadb'
-          mysql-version: '10.8'
+          mysql-version: '10.6'
           auto-start: false
       - name: Build database
         run: |
@@ -44,7 +44,7 @@ jobs:
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         with:
             token: ${{ secrets.GITHUB_TOKEN }}
             commit-message: Update database sql files

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/mariadb:10.10.3
+FROM docker.io/library/mariadb:10.6.19
 
 WORKDIR /ispyb
 

HISTORY.rst

@@ -5,6 +5,149 @@ History
 Unreleased / master
 -------------------
 
+4.10.0 (2025-11-20)
+-------------------
+
+Thank you so much, Karl! / Tusen takk, Karl!
+
+Lookups:
+
+- Add ``CLEM`` experiment type
+- Add ``FIB`` experiment type
+
+New or modified tables, columns, lookup values:
+
+- New table ``LaserParameters``
+- New table ``LaserPoint``
+- Add ``LASER`` to ``RobotAction`` ``actionType`` column
+
+4.9.0 (2025-10-01)
+-------------------
+
+New columns:
+
+- Add ``startDate`` and ``endDate`` columns to the ``Proposal`` table
+- Add ``parentContainerLocation`` column to the ``Container`` table
+
+Table permissions for the expeye role:
+
+- ``BLSampleImage``
+- ``Protein_has_PDB``
+- The ``Ligand`` tables
+
+Stored procedures:
+
+- Updated procedure ``update_container_status`` to allow more statuses
+- New procedure ``insert_screening_v2`` with relevant grants
+- In source file for procedure ``retrieve_containers_on_beamline_with_status``:
+
+  - Use ``CREATE OR REPLACE ... PROCEDURE`` syntax
+  - Update maximum string length for ``p_status`` parameter
+
+Lookups:
+
+- A new type "SWISSCI 2 Drop" was added to ``ContainerType``  
+
+4.8.0 (2025-07-17)
+-------------------
+
+New or modified tables, columns, lookup values:
+
+- New table ``Ligand``
+- New table ``BLSample_has_Ligand``
+- New table ``Ligand_has_PDB``
+- Add ``pixelLocationX`` and ``pixelLocationY`` columns to ``Tomogram`` table
+- Add ``cryoEM`` container type to ``ContainerType``
+- Add ``type`` enum column to ``DewarRegistry``
+- Add ``icatId`` column to ``BLSession``
+- Set ``recordTimeStamp`` column in ``BLSamplePosition`` to default to current time
+
+4.7.0 (2025-05-12)
+-------------------
+
+New or modified tables, columns, lookup values:
+
+- New table ``BLSamplePosition``
+- Add a ``blSampleId`` FK to the ``XrayCentringResult`` table
+- Add ``angularEfficiency`` and ``suggestedTilt`` columns to ``ParticleClassification``
+- Recreate FK constraints referencing ``LabContact(labContactId)`` with 'on delete set null
+- Make ``Shipping`` FK ``deliveryAgent_flightCodePersonId`` "on delete set null"
+
+Roles and permissions:
+
+- Add new grants for Expeye
+- Add new permissions for ``ispyb_propagation``
+- Add new permissions for ``ispyb_scripts_processing.sql``
+
+Stored procedures:
+
+- Also return sample locations for ``update_container_assign``
+
+4.6.0 (2025-03-06)
+-------------------
+
+New columns:
+
+- Add a ``dewarRegistryId`` FK to the ``Dewar`` table
+
+Data in lookup tables:
+
+- Add 'Atlas' experiment type
+
+Stored procedures:
+
+- Remove superfluous commit in procedure ``upsert_processing_program``
+- New procedure ``upsert_processing_program_v2`` with additional param for pipeline ID
+
+Permissions:
+
+- Add ``SELECT`` grant for Expeye role on table ``AutoProcIntegration``
+- Add ``EXECUTE`` grant for processing role on new procedure ``upsert_processing_program_v2``
+
+4.5.0 (2025-02-25)
+-------------------
+
+New or modified tables and columns:
+
+- Add ``purgedProcessedData`` boolean column to the ``BLSession`` table. This is
+a flag to indicate whether the ``processed`` folder in the associated visit
+directory has been purged.
+
+4.4.0 (2025-01-29)
+-------------------
+
+New or modified tables and columns:
+
+- Add ``binnedPixelSize`` column to the ``ParticleClassificationGroup`` table
+- Add ``gridSquareId`` column to the ``Tomogram`` table
+
+4.3.0 (2024-12-05)
+-------------------
+
+Note: The schema files now include a command in the first line to set the sanbox
+mode which is incompatible with older clients. For more info and to disable, see:
+https://mariadb.org/mariadb-dump-file-compatibility-change/
+
+Admin: Changes to GitHub workflows as 'main' is now the default branch (previously 'master')
+
+New or modified tables and columns:
+
+- Add ``processingPipelineId`` column to the ``AutoProcProgram`` table
+- Add new boolean column ``deleted`` to the ``AutoProcProgramAttachment`` table
+- Add unique index on ``BLSample(containerId, location, subLocation)``
+- Update the ``ProcessedTomogram`` table comment
+
+Roles and permissions:
+
+- Add new grants for Expeye and combine grants into two roles instead of three
+- Add roles and grants for MaxScale
+- Add role with grants for fragment screening
+
+Scripts and data changes:
+
+- Trim trailing '1' in the ``SpaceGroup`` table's ``spaceGroupShortName`` column
+- Add scripts that can delete ``DataCollection`` and ``DataCollectionGroup`` rows and referenced rows in other tables
+
 4.2.1 (2024-08-19)
 -------------------
 

README.md

@@ -2,19 +2,19 @@
 
 # ispyb-database
 
-This package provides everything needed to create the Diamond flavour of the ISPyB database schema. As the schema evolves the update scripts will also be published here, so other users of the schema should be able to stay up-to-date.
+This repository provides everything needed to create the Diamond flavour of the ISPyB database schema. As the schema evolves the update scripts will also be published here, so other users of the schema should be able to stay up-to-date.
 
 ## Requirements
 
-* Server and a client packages of MariaDB 10.3 or later.
+* Server and a client packages of MariaDB 10.6 or later.
 * A Unix-like OS with `bash` shell.
 * The pandoc package (only required to build documentation at the end of the `build.sh` script.)
 * If binary logging is enabled in the DB system, then execute this before importing the test schema: `SET GLOBAL log_bin_trust_function_creators=ON;`
 
 ## Installation
 
-1. Install MariaDB. See the Wiki for ideas on how to install in a particular environment.
-2. Copy `.my.example.cnf` to `.my.cnf` and then edit that file to set the `user` and `password`, e.g. `user = root` and use the `password` you set when securing. Optionally, you can also set e.g. `host` and `port`.
+1. Install MariaDB. The Wiki has ideas on how to install in a particular environment, including OS packages or containers via the provided Dockerfile. If you use the container approach with the provided Dockerfile, then steps 2 and 3 below are done for you automatically.
+2. Copy `.my.example.cnf` to `.my.cnf` and then edit that file to set the `user` and `password`, e.g. `user = root` and correspond `password`. Optionally, you can also set e.g. `host` and `port`.
 3. In a test environment run the `build.sh` file. This creates the database schema and applies the grants as described in the "Schema" and "Grants" sections below.
 
 ### Schema
@@ -94,8 +94,8 @@ mariadb ispyb < grants/ispyb_web.sql
 ## Useful scripts
 
 * `build.sh`: Creates a development `ispyb_build` database and imports all the schema and grants files into it.
-* `release.sh`: Makes a release, see the Releasing section.
-* `bin/backup.sh`: Makes a backup of the development database.
+* `release.sh`: Makes a release, see the `RELEASE.md` file for details.
+* `bin/backup.sh`: Makes a backup of the development database, producing four files: `tables.sql`, `routines.sql`, `lookups.sql`, `data.sql`.
 * `bin/missed_updates.sh`: Generates a list of files in the `schemas/ispyb/updates/` folder that haven't been applied.
 * `bin/export_session.sh`: Exports a given session's database rows to SQL files. These can then easily be imported into an empty ispyb database. This can be used e.g. to create a single-session ISPyB/SynchWeb instance if combined with all the relevant data files and processing files for the session, and an instance of SynchWeb.
 

build.sh

@@ -26,14 +26,18 @@ function exec_schema {
 
 function exec_grants {
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_acquisition.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_propagation.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_expeye.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_fragment_screening.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_import.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_processing.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_ro_nopii.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_scripts_processing.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_ssx_writer.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_touchscreen.sql
-  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_web.sql
   mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_web_verify_tests.sql
-  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_ssx_writer.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/ispyb_web.sql
+  mariadb --defaults-file=.my.cnf -D $1 < grants/maxscale.sql
 }
 
 function exec_missed_updates {

grants/ispyb_expeye.sql

@@ -8,11 +8,72 @@ CREATE ROLE IF NOT EXISTS ispyb_expeye_core_role;
 
 GRANT ispyb_propagation TO ispyb_expeye_core_role;
 
-CREATE ROLE ispyb_expeye_acquisition_role;
+CREATE ROLE IF NOT EXISTS ispyb_expeye_acquisition_role;
 
-GRANT SELECT, INSERT, UPDATE ON RobotAction TO ispyb_expeye_acquisition_role;
-GRANT SELECT, INSERT, UPDATE ON GridInfo TO ispyb_expeye_acquisition_role;
-GRANT SELECT, INSERT, UPDATE ON DataCollection TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Proposal TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSession TO ispyb_expeye_acquisition_role;
 GRANT SELECT, INSERT, UPDATE ON DataCollectionGroup TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON DataCollection TO ispyb_expeye_acquisition_role;
 GRANT SELECT, INSERT, UPDATE ON Position TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Shipping TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ShippingHasSession TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Project_has_Shipping TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON LabContact TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Dewar TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarLocation TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarLocationList TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarRegistry TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarRegistry_has_Proposal TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarReport TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON DewarTransportHistory TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Container TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerInspection TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerQueue TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerQueueSample TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerRegistry TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerRegistry_has_Proposal TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerReport TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerType TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON ContainerHistory TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT ON ProcessingPipeline TO ispyb_expeye_acquisition_role;
+GRANT SELECT ON ProcessingPipelineCategory TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSample TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE on BLSampleImage TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSampleGroup TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSampleGroup_has_BLSample TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSubSample TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Project_has_BLSample TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON BLSamplePosition TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT ON SpaceGroup TO ispyb_expeye_acquisition_role;
+GRANT SELECT ON Protein TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON Crystal TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON XFEFluorescenceSpectrum TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON DiffractionPlan TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT ON ComponentType TO ispyb_expeye_acquisition_role;
+GRANT SELECT ON ComponentSubType TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Component_has_SubType TO ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE ON RobotAction TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON GridInfo TO ispyb_expeye_acquisition_role;
+
 GRANT ispyb_expeye_acquisition_role TO ispyb_expeye_core_role;
+
+GRANT SELECT ON AutoProcProgram TO ispyb_expeye_core_role;
+GRANT SELECT, INSERT, UPDATE ON AutoProcProgramAttachment TO ispyb_expeye_core_role;
+GRANT SELECT ON AutoProcIntegration TO ispyb_expeye_core_role;
+
+GRANT SELECT ON ProcessingJob TO ispyb_expeye_core_role;
+
+GRANT SELECT, INSERT, UPDATE ON Ligand TO ispyb_expeye_core_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Ligand_has_PDB TO ispyb_expeye_core_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSample_has_Ligand TO ispyb_expeye_core_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Protein_has_PDB TO ispyb_expeye_core_role;