diff --git a/src/current/_includes/sidebar-data-v25.4.json b/src/current/_includes/sidebar-data-v25.4.json index ab270e056d9..2b7178876fe 100644 --- a/src/current/_includes/sidebar-data-v25.4.json +++ b/src/current/_includes/sidebar-data-v25.4.json @@ -11,6 +11,7 @@ {% include_cached v25.4/sidebar-data/feature-overview.json %}, {% include_cached v25.4/sidebar-data/resilience.json %}, {% include_cached v25.4/sidebar-data/connect-to-cockroachdb.json %}, + {% include_cached v25.4/sidebar-data/migrate-new.json %}, {% include_cached v25.4/sidebar-data/migrate.json %}, {% include_cached v25.4/sidebar-data/cloud-deployments.json %}, {% include_cached v25.4/sidebar-data/self-hosted-deployments.json %}, diff --git a/src/current/_includes/v25.4/sidebar-data/migrate-new.json b/src/current/_includes/v25.4/sidebar-data/migrate-new.json new file mode 100644 index 00000000000..41e84edb91a --- /dev/null +++ b/src/current/_includes/v25.4/sidebar-data/migrate-new.json @@ -0,0 +1,193 @@ +{ + "title": "Migrate NEW", + "is_top_level": true, + "items": [ + { + "title": "Overview", + "urls": [ + "/molt/migration-overview.html" + ] + }, + { + "title": "Migration Considerations", + "items": [ + { + "title": "Overview", + "urls": [ + "/molt/migration-considerations.html" + ] + }, + { + "title": "Migration Granularity", + "urls": [ + "/molt/migration-considerations-phases.html" + ] + }, + { + "title": "Continuous Replication", + "urls": [ + "/molt/migration-considerations-replication.html" + ] + }, + { + "title": "Data Transformation Strategy", + "urls": [ + "/molt/migration-considerations-transformation.html" + ] + }, + { + "title": "Validation Strategy", + "urls": [ + "/molt/migration-considerations-validation.html" + ] + }, + { + "title": "Rollback Plan", + "urls": [ + "/molt/migration-considerations-rollback.html" + ] + }, + { + "title": "Cutover Plan", + "urls": [ + "/molt/migration-considerations-cutover.html" + ] + } + ] + }, + { + "title": "MOLT Tools", + "items": [ + { + "title": "Schema Conversion Tool", + "urls": [ + "/cockroachcloud/migrations-page.html" + ] + }, + { + "title": "Fetch", + "urls": [ + "/molt/molt-fetch.html" + ] + }, + { + "title": "Replicator", + "urls": [ + "/molt/molt-replicator.html" + ] + }, + { + "title": "Verify", + "urls": [ + "/molt/molt-verify.html" + ] + } + ] + }, + { + "title": "Migration Walkthroughs", + "items": [ + { + "title": "Migration with Downtime", + "urls": [ + "/molt/migrate-bulk-load.html" + ] + }, + { + "title": "Near-zero Downtime Migration", + "urls": [ + "/molt/migrate-load-replicate.html" + ] + }, + { + "title": "Near-zero Downtime Migration with Failback", + "urls": [ + "/molt/migrate-failback.html" + ] + } + ] + }, + { + "title": "Third-Party Migration Tools", + "items": [ + { + "title": "AWS DMS", + "urls": [ + "/${VERSION}/aws-dms.html" + ] + }, + { + "title": "Qlik Replicate", + "urls": [ + "/${VERSION}/qlik.html" + ] + }, + { + "title": "Striim", + "urls": [ + "/${VERSION}/striim.html" + ] + }, + { + "title": "Oracle GoldenGate", + "urls": [ + "/${VERSION}/goldengate.html" + ] + }, + { + "title": "Debezium", + "urls": [ + "/${VERSION}/debezium.html" + ] + } + ] + }, + { + "title": "Migrate Data Types", + "items": [ + { + "title": "Migrate from CSV", + "urls": [ + "/${VERSION}/migrate-from-csv.html" + ] + }, + { + "title": "Migrate from Avro", + "urls": [ + "/${VERSION}/migrate-from-avro.html" + ] + }, + { + "title": "Migrate from Shapefiles", + "urls": [ + "/${VERSION}/migrate-from-shapefiles.html" + ] + }, + { + "title": "Migrate from OpenStreetMap", + "urls": [ + "/${VERSION}/migrate-from-openstreetmap.html" + ] + }, + { + "title": "Migrate from GeoJSON", + "urls": [ + "/${VERSION}/migrate-from-geojson.html" + ] + }, + { + "title": "Migrate from GeoPackage", + "urls": [ + "/${VERSION}/migrate-from-geopackage.html" + ] + }, + { + "title": "Import Performance Best Practices", + "urls": [ + "/${VERSION}/import-performance-best-practices.html" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/src/current/_includes/v25.4/sidebar-data/migrate.json b/src/current/_includes/v25.4/sidebar-data/migrate.json index 77b969311fb..fa93667573b 100644 --- a/src/current/_includes/v25.4/sidebar-data/migrate.json +++ b/src/current/_includes/v25.4/sidebar-data/migrate.json @@ -1,5 +1,5 @@ { - "title": "Migrate", + "title": "Migrate OLD", "is_top_level": true, "items": [ { diff --git a/src/current/images/molt/molt_flows_1.svg b/src/current/images/molt/molt_flows_1.svg new file mode 100644 index 00000000000..c730a6f84ca --- /dev/null +++ b/src/current/images/molt/molt_flows_1.svg @@ -0,0 +1,856 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/current/images/molt/molt_flows_2.svg b/src/current/images/molt/molt_flows_2.svg new file mode 100644 index 00000000000..d965814883e --- /dev/null +++ b/src/current/images/molt/molt_flows_2.svg @@ -0,0 +1,699 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/current/images/molt/molt_flows_3.svg b/src/current/images/molt/molt_flows_3.svg new file mode 100644 index 00000000000..fa8a9d22f72 --- /dev/null +++ b/src/current/images/molt/molt_flows_3.svg @@ -0,0 +1,736 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/current/molt/migrate-bulk-load.md b/src/current/molt/migrate-bulk-load.md index 666e5d1ac31..d99123fef5e 100644 --- a/src/current/molt/migrate-bulk-load.md +++ b/src/current/molt/migrate-bulk-load.md @@ -9,6 +9,12 @@ Perform a one-time bulk load of source data into CockroachDB. {% include molt/crdb-to-crdb-migration.md %} +## Migration sequence + +
+MOLT tooling overview +
+ {% include molt/molt-setup.md %} ## Start Fetch diff --git a/src/current/molt/migrate-to-cockroachdb.md b/src/current/molt/migrate-to-cockroachdb.md index a64832549da..13b28bd1cc3 100644 --- a/src/current/molt/migrate-to-cockroachdb.md +++ b/src/current/molt/migrate-to-cockroachdb.md @@ -11,8 +11,8 @@ MOLT Fetch supports various migration flows using [MOLT Fetch modes]({% link mol | Migration flow | Mode | Description | Best for | |---------------------------------------------------------------------|------------------------------|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| -| [Bulk load]({% link molt/migrate-bulk-load.md %}) | `--mode data-load` | Perform a one-time bulk load of source data into CockroachDB. | Testing, migrations with [planned downtime]({% link molt/migration-strategy.md %}#approach-to-downtime) | -| [Load and replicate]({% link molt/migrate-load-replicate.md %}) | MOLT Fetch + MOLT Replicator | Load source data using MOLT Fetch, then replicate subsequent changes using MOLT Replicator. | [Minimal downtime]({% link molt/migration-strategy.md %}#approach-to-downtime) migrations | +| [Bulk load]({% link molt/migrate-bulk-load.md %}) | `--mode data-load` | Perform a one-time bulk load of source data into CockroachDB. | Testing, migrations with [planned downtime]({% link molt/migration-considerations.md %}#permissible-downtime) | +| [Load and replicate]({% link molt/migrate-load-replicate.md %}) | MOLT Fetch + MOLT Replicator | Load source data using MOLT Fetch, then replicate subsequent changes using MOLT Replicator. | [Minimal downtime]({% link molt/migration-considerations.md %}#permissible-downtime) migrations | | [Resume replication]({% link molt/migrate-resume-replication.md %}) | `--mode replication-only` | Resume replication from a checkpoint after interruption. | Resuming interrupted migrations, post-load sync | | [Failback]({% link molt/migrate-failback.md %}) | `--mode failback` | Replicate changes from CockroachDB back to the source database. | [Rollback]({% link molt/migrate-failback.md %}) scenarios | diff --git a/src/current/molt/migration-considerations-cutover.md b/src/current/molt/migration-considerations-cutover.md new file mode 100644 index 00000000000..eac110f84d2 --- /dev/null +++ b/src/current/molt/migration-considerations-cutover.md @@ -0,0 +1,8 @@ +--- +title: Cutover Plan +summary: Learn about the different approaches to cutover, and how to think about this for your migration. +toc: true +docs_area: migrate +--- + +TBD \ No newline at end of file diff --git a/src/current/molt/migration-considerations-phases.md b/src/current/molt/migration-considerations-phases.md new file mode 100644 index 00000000000..46c36907491 --- /dev/null +++ b/src/current/molt/migration-considerations-phases.md @@ -0,0 +1,76 @@ +--- +title: Migration Granularity +summary: Learn how to think about phased data migration, and whether or not to approach your migration in phases. +toc: true +docs_area: migrate +--- + +You may choose to migrate all of your data into a CockroachDB cluster at once. However, for larger data stores it's recommended that you migrate data in separate phases. This can help break the migration down into manageable slices, and it can help limit the effects of migration difficulties. + +This page explains when to choose each approach, how to define phases, and how to use MOLT tools effectively in either context. + +In general: + +- Choose to migrate your data **all at once** if your data volume is modest, if you want to minimize migration complexity, or if you don't mind taking on a greater risk of something going wrong. + +- Choose a **phased migration** if your data volume is large, especially if you can naturally partition workload by tenant, service/domain, table/shard, geography, or time. A phased migration helps to reduce risk by limiting the workloads that would be adversely affected by a migration failure. It also helps to limit the downtime per phase, and allows the application to continue serving unaffected subsets of the data during the migration of a phase. + +## How to divide migrations into phases + +Here are some common ways to divide migrations: + +* **Per-tenant**: Multi-tenant apps route traffic and data per customer/tenant. Migrate a small cohort first (canary), then progressively larger cohorts. This aligns with access controls and isolates blast radius. + +* **Per-service/domain**: In microservice architectures, migrate data owned by a service or domain (e.g., billing, catalog) and route only that service to CockroachDB while others continue on the source. Requires clear data ownership and integration contracts. + +* **Per-table or shard**: Start with non-critical tables, large-but-isolated tables, or shard ranges. For monolith schemas, you can still phase by tables with few foreign-key dependencies and clear read/write paths. + +* **Per-region/market**: If traffic is regionally segmented, migrate one region/market at a time and validate latency, capacity, and routing rules before expanding. + +Tips for picking slices: + +- Prefer slices with clear routing keys (tenant_id, region_id) to simplify cutover and verification. + +- Start with lower-impact slices to exercise the migration process before migrating high-value cohorts. + +## Tradeoffs + +| | All at once | Phased | +|---|---|---| +| Downtime | A single downtime window, but it affects the whole database | Multiple short windows, each with limited impact | +| Risk | Higher blast radius if issues surface post-cutover | Lower blast radius, issues confined to a slice | +| Complexity | Simpler orchestration, enables a single cutover | More orchestration, repeated verify and cutover steps | +| Validation | One-time, system-wide | Iterative per slice; faster feedback loops | +| Timeline | Shorter migration time | Longer calendar time but safer path | +| Best for | Small/medium datasets, simple integrations | Larger datasets, data with natural partitions or multiple tenants, risk-averse migrations | + +## MOLT toolkit support + +Phased and unphased migrations are both supported natively by MOLT. + +By default, [MOLT Fetch]({% link molt/molt-fetch.md %}) moves all data from the source database to CockroachDB. However, you can use the `--schema-filter`, `--table-filter`, and `--filter-path` flags to selective migrate data from the source to the target. Learn more about [schema and table selection]({% link molt/molt-fetch.md %}#schema-and-table-selection) and [selective data movement]({% link molt/molt-fetch.md %}#selective-data-movement), both of which can enable a phased migration. + +Similarly, you can use [MOLT Verify]({% link molt/molt-verify.md %})'s `--schema-filter` and `--table-filter` flags to run validation checks on subsets of the data in your source and target databases. In a phased migration, you will likely want to verify data at the end of each migration phase, rather than at the end of the entire migration. + +[MOLT Replicator]({% link molt/molt-replicator.md %}) replicates full tables by default. If you choose to combine phased migration with [continuous replication]({% link molt/migration-considerations-replication.md %}), you will either need to select phases that include whole tables, or else use [userscripts]({% link molt/molt-replicator.md %}#flags) to select rows to replicate. + +## Example sequences + +### Migrating all data at once + +
+MOLT tooling overview +
+ +### Phased migration + +
+MOLT tooling overview +
+ +## See Also + +- [Migration Overview]({% link molt/migration-overview.md %}) +- [Migration Considerations]({% link molt/migration-considerations.md %}) +- [Continuous Replication]({% link molt/migration-considerations-replication.md %}) +- [MOLT Fetch]({% link molt/molt-fetch.md %}) diff --git a/src/current/molt/migration-considerations-replication.md b/src/current/molt/migration-considerations-replication.md new file mode 100644 index 00000000000..e90dc40e996 --- /dev/null +++ b/src/current/molt/migration-considerations-replication.md @@ -0,0 +1,8 @@ +--- +title: Continuous Replication +summary: Learn about continuous replication, and how it might be useful for a data migration. +toc: true +docs_area: migrate +--- + +TBD \ No newline at end of file diff --git a/src/current/molt/migration-considerations-rollback.md b/src/current/molt/migration-considerations-rollback.md new file mode 100644 index 00000000000..da38fa50b86 --- /dev/null +++ b/src/current/molt/migration-considerations-rollback.md @@ -0,0 +1,8 @@ +--- +title: Rollback Plan +summary: Learn why it might be useful to have a rollback plan in case something goes wrong in the middle of a migration. +toc: true +docs_area: migrate +--- + +TBD \ No newline at end of file diff --git a/src/current/molt/migration-considerations-transformation.md b/src/current/molt/migration-considerations-transformation.md new file mode 100644 index 00000000000..b911679ed09 --- /dev/null +++ b/src/current/molt/migration-considerations-transformation.md @@ -0,0 +1,8 @@ +--- +title: Data Transformation Strategy +summary: Learn about the different approaches to applying data transformations during a migration. +toc: true +docs_area: migrate +--- + +TBD \ No newline at end of file diff --git a/src/current/molt/migration-considerations-validation.md b/src/current/molt/migration-considerations-validation.md new file mode 100644 index 00000000000..dc5d50e7e39 --- /dev/null +++ b/src/current/molt/migration-considerations-validation.md @@ -0,0 +1,8 @@ +--- +title: Validation Strategy +summary: Learn what different validation strategies can be used when migrating data. +toc: true +docs_area: migrate +--- + +TBD \ No newline at end of file diff --git a/src/current/molt/migration-considerations.md b/src/current/molt/migration-considerations.md new file mode 100644 index 00000000000..07f6151b71f --- /dev/null +++ b/src/current/molt/migration-considerations.md @@ -0,0 +1,74 @@ +--- +title: Migration Considerations +summary: Learn what to consider when making high-level decisions about a migration. +toc: true +docs_area: migrate +--- + +When planning a migration to CockroachDB, you need to make several high-level decisions that will shape your migration approach. This page provides an overview of key migration variables and the factors that influence them. Each variable has multiple options, and the combination you choose will largely define your migration strategy. + +For detailed migration sequencing and tool usage, see [Migration Overview]({% link molt/migration-overview.md %}). For detailed planning guidance, see [Migration Strategy]({% link molt/migration-strategy.md %}). + +## Migration variables + +Learn more about each migration variable by clicking the links in the left-hand column. + +| Variable | Description | Options | +|---|---|---| +| [**Migration granularity**]({% link molt/migration-considerations-phases.md %}) | Do you want to migrate all of your data at once, or do you want to split your data up into phases and migrate one phase at a time? | - All at once
- Phased | +| [**Continuous replication**]({% link molt/migration-considerations-replication.md %}) | After the initial data load (or after the initial load of each phase), do you want to stream further changes to that data from the source to the target? | - Bulk load only
- Continuous replication | +| [**Data transformation strategy**]({% link molt/migration-considerations-transformation.md %}) | If there are discrepancies between the source and target schema, how will you define those data transformations, and when will those transformations occur? | - No data transformation
- Transform at source
- Tranform in flight
- Transform at target | +| [**Validation strategy**]({% link molt/migration-considerations-validation.md %}) | How and when will you verify that the data in CockroachDB matches the source database? | - After initial load
During sync
After cutover
QA sign-off | +| [**Rollback plan**]({% link molt/migration-considerations-rollback.md %}) | What approach will you use to roll back the migration if issues arise during or after cutover? | - Dual-write
- Bidirectional replication
- Failback (reverse replication)
- Manual reconciliation | +| [**Cutover strategy**]({% link molt/migration-considerations-cutover.md %}) | How will you transition application traffic from the source database to CockroachDB? | - Big-bang cutover
- Minimal-downtime cutover (with replication)
- Progressive cutover (per-slice) | + +The combination of these variables largely defines your migration approach. While you'll typically choose one primary option for each variable, some migrations may involve a hybrid approach depending on your specific requirements. + +## Factors to consider + +When deciding on the options for each migration variable, consider the following business and technical requirements: + +### Permissible downtime + +How much downtime can your application tolerate during the migration? This is one of the most critical factors in determining your migration approach, and it may influence your choices for [migration granularity]({% link molt/migration-considerations-phases.md %}), [continuous replication]({% link molt/migration-considerations-replication.md %}), and [cutover strategy]({% link molt/migration-considerations-cutover.md %}). + +- **Planned downtime** is made known to your users in advance. It involves taking the application offline, conducting the migration, and bginging the application back online on CockroachDB. + + To succeed, you should estimate the amount of downtime required to migrate your data, and ideally schedule the downtime outside of peak hours. Scheduling downtime is easiest if your application traffic is "periodic", meaning that it varies by the time of day, day of week, or day of month. + + If you can support planned downtime, you may want to migrate your data [all at once]({% link molt/migration-considerations-phases.md %}), and _without_ [continuous replication]({% link molt/migration-considerations-replication.md %}). + +- **Minimal downtime** impacts as few customers as possible, ideally without impacting their regular usage. If your application is intentionally offline at certain times (e.g., outside business hours), you can migrate the data without users noticing. Alternatively, if your application's functionality is not time-sensitive (e.g., it sends batched messages or emails), you can queue requests while the system is offline and process them after completing the migration to CockroachDB. + +- **Near-zero downtime** is necessary for mission-critical applications. For these migrations, consider cutover strategies that keep applications online for as long as possible, and which utilize [continuous replication]({% link molt/migration-considerations-replication.md %}). + +In addition to downtime duration, consider whether your application could support windows of **reduced functionality** in which some, but not all, application functionality is brought offline. For example, you can disable writes but not reads while you migrate the application data, and queue data to be written after completing the migration. + +### Migration timeframe and allowable complexity + +When do you need to complete the migration? How many team members can be allocated for this effort? How much complex orchestration can your team manage? These factors may influence your choices for [migration granularity]({% link molt/migration-considerations-phases.md %}), [continuous replication]({% link molt/migration-considerations-replication.md %}), and [cutover strategy]({% link molt/migration-considerations-cutover.md %}). + +- Migrations with a short timeline, or which cannot accommodate high complexity, may want to migrate data [all at once]({% link molt/migration-considerations-phases.md %}), without utilizing [continuous replication]({% link molt/migration-considerations-replication.md %}), and requiring [manual reconciliation]({% link molt/migration-considerations-rollback.md %}) in the event of migration failure. + +- Migrations with a long timeline, or which can accomodate complexity, may want to migrate data [in phases]({% link molt/migration-considerations-phases.md %}). If the migration requires minimal downtime, these migrations may also want to utilize [continuous replication]({% link molt/migration-considerations-replication.md %}). If the migration is low in risk-tolerance, these migrations may also want to enable [failback]({% link molt/migration-considerations-rollback.md %}). + +### Risk tolerance + +How much risk is your organization willing to accept during the migration? This may influence your choices for [migration granularity]({% link molt/migration-considerations-phases.md %}), [validation strategy]({% link molt/migration-considerations-validation.md %}), and [rollback plan]({% link molt/migration-considerations-rollback.md %}). + +- Risk-averse migrations should prefer [phased migrations]({% link molt/migration-considerations-phases.md %}) that limit the blast radius of any issues. Start with low-risk slices (e.g., a small cohort of tenants or a non-critical service), [validate thoroughly]({% link molt/migration-considerations-validation.md %}), and progressively expand to higher-value workloads. These migrations may also prefer [rollback plans]({% link molt/migration-considerations-rollback.md %}) that enable quick recovery in the event of migration issues. + +- For risk-tolerant migrations, it may be acceptable to migrate [all of your data at once]({% link molt/migration-considerations-phases.md %}). Less stringent [validation strategies]({% link molt/migration-considerations-validation.md %}) and [manual reconciliation]({% link molt/migration-considerations-rollback.md %}) in the event of a migration failure may also be acceptable. + +___ + +These above factors are only a subset of all of what you'll want to consider in the decision-making about your CockroachDB migration, along with your specific business requirements and technical constraints. It's recommended that you document these decisions and the reasoning behind them as part of your [migration plan]({% link molt/migration-strategy.md %}#develop-a-migration-plan). + +## See also + +- [Migration Overview]({% link molt/migration-overview.md %}) +- [Migration Strategy]({% link molt/migration-strategy.md %}) +- [Bulk vs. Phased Migration]({% link molt/migration-considerations-phases.md %}) +- [MOLT Fetch]({% link molt/molt-fetch.md %}) +- [MOLT Replicator]({% link molt/molt-replicator.md %}) +- [MOLT Verify]({% link molt/molt-verify.md %}) diff --git a/src/current/molt/migration-overview.md b/src/current/molt/migration-overview.md index 163dc26d8aa..800da53cab7 100644 --- a/src/current/molt/migration-overview.md +++ b/src/current/molt/migration-overview.md @@ -5,34 +5,47 @@ toc: true docs_area: migrate --- -The MOLT (Migrate Off Legacy Technology) toolkit enables safe, minimal-downtime database migrations to CockroachDB. MOLT combines schema transformation, distributed data load, continuous replication, and row-level validation into a highly configurable workflow that adapts to diverse production environments. +A migration involves transfering data from a pre-existing **source** database onto a **target** CockroachDB cluster. Migrating data is a complex, multi-step process, and a data migration can take many different forms depending on your specific business and technical constraints. + +Cockroach Labs provides a MOLT (Migrate Off Legacy Technology) toolkit to aid in migrations. This page provides an overview of the following: - Overall [migration sequence](#migration-sequence) - [MOLT tools](#molt-tools) -- Supported [migration flows](#migration-flows) ## Migration sequence -{{site.data.alerts.callout_success}} -Before you begin the migration, review [Migration Strategy]({% link molt/migration-strategy.md %}). -{{site.data.alerts.end}} - A migration to CockroachDB generally follows this sequence: -MOLT tooling overview + + +1. **Assess and discover**: Inventory the source database, flag unsupported features, make a migration plan. +1. **Prepare the environment**: Configure networking, users and permissions, bucket locations, replication settings, and more. +1. **Convert the source schema**: Generate CockroachDB-compatible [DDL]({% link {{ site.current_cloud_version }}/sql-statements.md %}#data-definition-statements). Apply the converted schema to the target database. Drop constraints and indexes to facilitate data load. +1. **Load data into CockroachDB**: Bulk load the source data into the CockroachDB cluster. +1. **Finalize target schema**: Recreate indexes or constraints on CockroachDB that you previously dropped to facilitate data load. +1. **_(Optional)_ Replicate ongoing changes**: Keep CockroachDB in sync with the source. This may be necessary for migrations that minimize downtime. +1. **Stop application traffic**: Limit user read/write traffic to the source database. _This begins application downtime._ +1. **Verify data consistency**: Confirm that the CockroachDB data is consistent with the source. +1. **_(Optional)_ Enable failback**: Replicate data from the target back to the source, enabling a reversion to the source database in the event of migration failure. +1. **Cut over application traffic**: Resume normal application use, with the CockroachDB cluster as the target database. _This ends application downtime._ -1. Prepare the source database: Configure users, permissions, and replication settings as needed. -1. Convert the source schema: Use the [Schema Conversion Tool]({% link cockroachcloud/migrations-page.md %}) to generate CockroachDB-compatible [DDL]({% link {{ site.current_cloud_version }}/sql-statements.md %}#data-definition-statements). Apply the converted schema to the target database. Drop constraints and indexes to facilitate data load. -1. Load data into CockroachDB: Use [MOLT Fetch]({% link molt/molt-fetch.md %}) to bulk-ingest your source data. -1. (Optional) Verify consistency before replication: Use [MOLT Verify]({% link molt/molt-verify.md %}) to confirm that the data loaded into CockroachDB is consistent with the source. -1. Finalize target schema: Recreate indexes or constraints on CockroachDB that you previously dropped to facilitate data load. -1. Replicate ongoing changes: Enable continuous replication with [MOLT Replicator]({% link molt/molt-replicator.md %}) to keep CockroachDB in sync with the source. -1. Verify consistency before cutover: Use [MOLT Verify]({% link molt/molt-verify.md %}) to confirm that the CockroachDB data is consistent with the source. -1. Cut over to CockroachDB: Redirect application traffic to the CockroachDB cluster. +The MOLT (Migrate Off Legacy Technology) toolkit enables safe, minimal-downtime database migrations to CockroachDB. MOLT combines schema transformation, distributed data load, continuous replication, and row-level validation into a highly configurable workflow that adapts to diverse production environments. -For more details, refer to [Migration flows](#migration-flows). + +
+MOLT tooling overview +
## MOLT tools @@ -87,7 +100,7 @@ The [MOLT Schema Conversion Tool]({% link cockroachcloud/migrations-page.md %}) [MOLT Fetch]({% link molt/molt-fetch.md %}) performs the initial data load to CockroachDB. It supports: -- [Multiple migration flows](#migration-flows) via `IMPORT INTO` or `COPY FROM`. +- Multiple migration flows via `IMPORT INTO` or `COPY FROM`. - Data movement via [cloud storage, local file servers, or direct copy]({% link molt/molt-fetch.md %}#data-path). - [Concurrent data export]({% link molt/molt-fetch.md %}#best-practices) from multiple source tables and shards. - [Schema transformation rules]({% link molt/molt-fetch.md %}#transformations). @@ -110,7 +123,11 @@ The [MOLT Schema Conversion Tool]({% link cockroachcloud/migrations-page.md %}) - Column definition. - Row-level data. -## Migration flows + -### Bulk load +## Migration variables -For migrations that tolerate downtime, use MOLT Fetch in `data-load` mode to perform a one-time bulk load of source data into CockroachDB. Refer to [Bulk Load]({% link molt/migrate-bulk-load.md %}). +You must decide how you want your migration to handle each of the following variables. These decisions will depend on your specific business and technical considerations. The MOLT toolkit supports any set of decisions made for the [supported source databases](#molt-tools). -### Migrations with minimal downtime +### Migration granularity -To minimize downtime during migration, use MOLT Fetch for initial data loading followed by MOLT Replicator for continuous replication. Instead of loading all data during a planned downtime window, you can run an initial load followed by continuous replication. Writes are paused only briefly to allow replication to drain before the final cutover. The duration of this pause depends on the volume of write traffic and the replication lag between the source and CockroachDB. +You may choose to migrate all of your data into a CockroachDB cluster at once. However, for larger data stores it's recommended that you migrate data in separate phases. This can help break the migration down into manageable slices, and it can help limit the effects of migration difficulties. -Refer to [Load and Replicate]({% link molt/migrate-load-replicate.md %}) for detailed instructions. +### Continuous replication -### Recovery and rollback strategies +After data is migrated from the source into CockroachDB, you may choose to continue streaming changes to that source data from the source to the target. This is important for migrations that aim to minimize application downtime, as they may require the source database to continue receiving writes until application traffic is fully cut over to CockroachDB. -If the migration is interrupted or cutover must be aborted, MOLT Replicator provides safe recovery options: +### Data transformation strategy + +If there are discrepencies between the source and target schemas, the rules that determine necessary data transformations need to be defined. These transformations can be applied in the source database, in flight, or in the target database. + +### Validation strategy + +There are several different ways of verifying that the data in the source and the target match one another. You must decide what validation checks you want to perform, and when in the migration process you want to perform them. + +### Rollback plan + +Until the migration is complete, migration failures may make you decide to roll back application traffic entirely to the source database. You may therefore need a way of keeping the source database up to date with new writes to the target. This is especially important for risk-averse migrations that aim to minimize downtime. + +### Cutover plan + +*Cutover* is the process of switching application traffic from the source database to CockroachDB. + +There are many different approaches to cutover. It can happen all at once, it can occur in multiple steps (in tandem with different [migration phases](#migration-granularity)). It's possible to cut over read and write traffic at different times. The possibilities are varied. + +The decision of how to cut over application traffic is closely linked with many of the other choices above. + +--- -- Resume a previously interrupted replication stream. Refer to [Resume Replication]({% link molt/migrate-resume-replication.md %}). -- Use failback mode to reverse the migration, synchronizing changes from CockroachDB back to the original source. This ensures data consistency on the source so that you can retry the migration later. Refer to [Migration Failback]({% link molt/migrate-failback.md %}). +[Learn more about the different migration variables]({% link molt/migration-considerations.md %}), how you should consider the different options for each variable, and how to use the MOLT toolkit for each variable. ## See also diff --git a/src/current/molt/migration-strategy.md b/src/current/molt/migration-strategy.md index eb2b46f65f5..b6e76d0c14f 100644 --- a/src/current/molt/migration-strategy.md +++ b/src/current/molt/migration-strategy.md @@ -10,10 +10,9 @@ A successful migration to CockroachDB requires planning for downtime, applicatio This page outlines key decisions, infrastructure considerations, and best practices for a resilient and repeatable high-level migration strategy: - [Develop a migration plan](#develop-a-migration-plan). -- Evaluate your [downtime approach](#approach-to-downtime). - [Size the target CockroachDB cluster](#capacity-planning). - Implement [application changes](#application-changes) to address necessary [schema changes](#schema-design-best-practices), [transaction contention](#handling-transaction-contention), and [unimplemented features](#unimplemented-features-and-syntax-incompatibilities). -- [Prepare for migration](#prepare-for-migration) by running a [pre-mortem](#run-a-migration-pre-mortem), setting up [metrics](#set-up-monitoring-and-alerting), [loading test data](#load-test-data), [validating application queries](#validate-queries) for correctness and performance, performing a [migration dry run](#perform-a-dry-run), and reviewing your [cutover strategy](#cutover-strategy). +- [Prepare for migration](#prepare-for-migration) by running a [pre-mortem](#run-a-migration-pre-mortem), setting up [metrics](#set-up-monitoring-and-alerting), [loading test data](#load-test-data), [validating application queries](#validate-queries) for correctness and performance, performing a [migration dry run](#perform-a-dry-run), and reviewing your cutover strategy. {% assign variable = value %} {{site.data.alerts.callout_success}} For help migrating to CockroachDB, contact our sales team. @@ -31,20 +30,6 @@ Consider the following as you plan your migration: Create a document summarizing the migration's purpose, technical details, and team members involved. -## Approach to downtime - -It's important to fully [prepare the migration](#prepare-for-migration) in order to be certain that the migration can be completed successfully during the downtime window. - -- *Planned downtime* is made known to your users in advance. Once you have [prepared for the migration](#prepare-for-migration), you take the application offline, [conduct the migration]({% link molt/migration-overview.md %}), and bring the application back online on CockroachDB. To succeed, you should estimate the amount of downtime required to migrate your data, and ideally schedule the downtime outside of peak hours. Scheduling downtime is easiest if your application traffic is "periodic", meaning that it varies by the time of day, day of week, or day of month. - - Migrations with planned downtime are only recommended if you can complete the bulk data load (e.g., using the MOLT Fetch [`data-load` mode]({% link molt/molt-fetch.md %}#fetch-mode)) within the downtime window. Otherwise, you can [minimize downtime using continuous replication]({% link molt/migration-overview.md %}#migrations-with-minimal-downtime). - -- *Minimal downtime* impacts as few customers as possible, ideally without impacting their regular usage. If your application is intentionally offline at certain times (e.g., outside business hours), you can migrate the data without users noticing. Alternatively, if your application's functionality is not time-sensitive (e.g., it sends batched messages or emails), you can queue requests while the system is offline and process them after completing the migration to CockroachDB. - - MOLT enables [migrations with minimal downtime]({% link molt/migration-overview.md %}#migrations-with-minimal-downtime), using [MOLT Replicator]({% link molt/molt-replicator.md %}) for continuous replication of source changes to CockroachDB. - -- *Reduced functionality* takes some, but not all, application functionality offline. For example, you can disable writes but not reads while you migrate the application data, and queue data to be written after completing the migration. - ## Capacity planning To size the target CockroachDB cluster, consider your data volume and workload characteristics: @@ -110,7 +95,7 @@ Based on the error budget you [defined in your migration plan](#develop-a-migrat ### Load test data -It's useful to load test data into CockroachDB so that you can [test your application queries](#validate-queries). Refer to [Migration flows]({% link molt/migration-overview.md %}#migration-flows). +It's useful to load test data into CockroachDB so that you can [test your application queries](#validate-queries). MOLT Fetch [supports both `IMPORT INTO` and `COPY FROM`]({% link molt/molt-fetch.md %}#data-load-mode) for loading data into CockroachDB: @@ -147,7 +132,7 @@ To further minimize potential surprises when you conduct the migration, practice Performing a dry run is highly recommended. In addition to demonstrating how long the migration may take, a dry run also helps to ensure that team members understand what they need to do during the migration, and that changes to the application are coordinated. -## Cutover strategy + ## See also