Replicating Big tables

[QuiosaEvaristo]

Good day,

I am trying to make full load of a big table with 556 GB of size, it starts normally replicating but at some point of time it stops and shows the error bellow.

I am new in qlik replicate world and need your support to find the best approach to make the replication of this big table run successfully

[john_wang]

Hello @QuiosaEvaristo ,

Welcome to Qlik Community!

Would you please share what's the target endpoint type?

And please check the article: ORA-01555: Snapshot too old.

Hope this helps.

John.

[QuiosaEvaristo]

Hello @john_wang ,

Thanks for your prompt reply the endpoint type is SQL Server

[john_wang · Accepted solution]

Hello @QuiosaEvaristo ,

Oracle source replicates to SQL Server target task supports the Parallel Load. This is the first recommended approach.

Good luck,

John.

[Vegy]

We use Parrallel Load for large tables @QuiosaEvaristo , different source but it's a standard practice really. We've found it incredibly useful. If you have the bandwidth, i.e. you're allowed say 10 connections to the source with the same throughput as 1 connection, you may find it's actually faster than 10x too.

[QuiosaEvaristo · Accepted solution]

Hello,

I have solved the issue using parallel loading.
I have partitioned the table in 64 chunks then set it in Qlik replicate parallel according to the ranging I got from the chunks created.

I could conduct full loading of 619.000.000 in 9 hous.

Here you have the steps used

1 - I have created partitioned the tables in buckets(Chunks) in the source db (ORACLE)

WITH sampled AS (
SELECT rowID,
NTILE(64) OVER (ORDER BY rowID) AS bucket
FROM T24USER.FBNK_STMT_ENTRY SAMPLE BLOCK (2)
),
boundaries AS (
SELECT bucket, rowID,
ROW_NUMBER() OVER (PARTITION BY bucket ORDER BY rowID) AS rn
FROM sampled
),
b AS (
SELECT bucket, rowID
FROM boundaries
WHERE rn = 1
)
SELECT
'{"values":' || CHR(9) || '["' || rowID || '"]},' AS json_line,
'CREATE PARTITION FUNCTION [myTable_rowID] (varchar(127))' || CHR(13)||CHR(10) ||
'AS RANGE RIGHT FOR VALUES (' || CHR(13)||CHR(10) ||
LISTAGG(' ''' || rowID || '''', ',' || CHR(13)||CHR(10))
WITHIN GROUP (ORDER BY rowID COLLATE BINARY_CI) OVER () || CHR(13)||CHR(10) ||
');' || CHR(13)||CHR(10) ||
'GO' || CHR(13)||CHR(10) ||
'CREATE PARTITION SCHEME [PS_FBNK_STMT_ENTRY_rowID]' || CHR(13)||CHR(10) ||
'AS PARTITION [myTable_rowID]' || CHR(13)||CHR(10) ||
'AS PARTITION [myTable_rowID]' || CHR(13)||CHR(10) ||
'ALL TO ([SECONDARY]);' || CHR(13)||CHR(10) ||
'GO' AS mssql_ddl
FROM b
ORDER BY bucket;

2 - Creation of partition function with the rangings values extracted from the query result above

CREATE PARTITION FUNCTION [PF_FBNK_STMT_ENTRY_rowID] (varchar(64))
AS RANGE RIGHT FOR VALUES (
'164064869182220.020001',
'184752163646034.000002',
'186092978855431.040002',
'187427854537895.650002',
'188585350040012.120001',
'189608375137241.760002',
'190512328174404.480001',
'191456759132598.800001',
'192330981501573.010001',
'193115129447695.020004',
'193825602451683.040002',
'194488093813298.070004',
'195134133829136.000002',
'195742599827058.000002',
'196328413876098.010003',
'196880662121964.050001',
'197363731119817.010001',
'197871384917268.010001',
'198361834541441.040002',
'198805244682515.010001',
'199272286638593.040001',
'199703441426393.080002',
'200167432052681.090002',
'200591823812392.020001',
'200968308317761.030001',
'201383975716480.040002',
'201786416914740.030001',
'202164947034881.040001',
'202564937237470.040001',
'202947777139636.020002',
'203311526028104.010002',
'203660546417714.030001',
'204013619619220.050001',
'204352380837189.050001',
'204694947715562.000001',
'205044958345323.020002',
'205376653885567.000002',
'205708194566217.040001',
'206019246459845.000002',
'206328628269661.020001',
'206628221303161.000001',
'206927716356124.050002',
'207227120176009.090001',
'207500675828738.040001',
'207816256311462.000001',
'208130807246103.000001',
'208483520084669.020001',
'208807125005906.020002',
'209121412207743.020001',
'209437311304962.010001',
'209746953505883.010001',
'210047063105221.010002',
'210353029456519.000001',
'210650263506847.000001',
'210938957763803.000001',
'211225800004018.000001',
'211502130806494.000001',
'211737573865694.000001',
'211996233748044.010001',
'212254253769320.010002',
'212512512505708.000001',
'212764916414839.010002',
'213028782860657.000001',
'213267278505776.020002'
);

GO
CREATE PARTITION SCHEME [myTable_rowID]
AS PARTITION [PF_FBNK_STMT_ENTRY_rowID]

ALL TO ([SECONDARY]);
GO

3 - Set parallel loading with one segment in qlik replicate then export the task  and edit it as follow and imported after this step with repload the target and the full loading was done successfully

{
"name": "PRD_DB__LSS_Target_myTable__2026-06-11--11-29-12-323537",
"cmd.replication_definition": {
"tasks": [
{
"task": {
"name": "PRD_DB__LSS_Target_myTable",
"description": "myTable TABLE",
"source_name": "PRD_DB_-LSS-SRC",
"target_names": [
"PRD-AO-LSS-TargetDB"
]
},
"source": {
"rep_source": {
"source_name": "PRD_DB_-LSS-SRC",
"database_name": "PRD_DB_-LSS-SRC"
},
"source_tables": {
"name": "PRD_DB_-LSS-SRC",
"explicit_included_tables": [
{
"owner": "MyUser",
"name": "myTable",
"estimated_size": 71467161,
"orig_db_id": 110511,
"sapdb_def_ext_for_logstream": {
"$type": "SapdbOverLogstreamTableDefExt"
}
}
]
}
},
"targets": [
{
"rep_target": {
"target_name": "PRD-AO-LSS-TargetDB",
"target_state": "DISABLED",
"database_name": "PRD-AO-LSS-TargetDB"
}
}
],
"manipulations": [
{
"name": "MyUser.myTable",
"table_manipulation": {
"owner": "MyUser",
"name": "myTable",
"transform_columns": [
{
"column_name": "rowID",
"action": "KEEP",
"new_data_type": "kAR_DATA_TYPE_STR",
"length": 64,
"new_sub_data_type": "KAR_SUB_DATA_TYPE_UNSPECIFIED"
}
],
"source_table_settings": {
"unload_segments": {
"segments_type": "RANGES",
"ranges": {
"column_names": [
"rowID"
],
"split_points": [
{
"values": [
"164064869182220.020001"
]
},
{
"values": [
"184752163646034.000002"
]
},
{
"values": [
"186092978855431.040002"
]
},
{
"values": [
"187427854537895.650002"
]
},
{
"values": [
"188585350040012.120001"
]
},
{
"values": [
"189608375137241.760002"
]
},
{
"values": [
"190512328174404.480001"
]
},
{
"values": [
"191456759132598.800001"
]
},
{
"values": [
"192330981501573.010001"
]
},
{
"values": [
"193115129447695.020004"
]
},
{
"values": [
"193825602451683.040002"
]
},
{
"values": [
"194488093813298.070004"
]
},
{
"values": [
"195134133829136.000002"
]
},
{
"values": [
"195742599827058.000002"
]
},
{
"values": [
"196328413876098.010003"
]
},
{
"values": [
"196880662121964.050001"
]
},
{
"values": [
"197363731119817.010001"
]
},
{
"values": [
"197871384917268.010001"
]
},
{
"values": [
"198361834541441.040002"
]
},
{
"values": [
"198805244682515.010001"
]
},
{
"values": [
"199272286638593.040001"
]
},
{
"values": [
"199703441426393.080002"
]
},
{
"values": [
"200167432052681.090002"
]
},
{
"values": [
"200591823812392.020001"
]
},
{
"values": [
"200968308317761.030001"
]
},
{
"values": [
"201383975716480.040002"
]
},
{
"values": [
"201786416914740.030001"
]
},
{
"values": [
"202164947034881.040001"
]
},
{
"values": [
"202564937237470.040001"
]
},
{
"values": [
"202947777139636.020002"
]
},
{
"values": [
"203311526028104.010002"
]
},
{
"values": [
"203660546417714.030001"
]
},
{
"values": [
"204013619619220.050001"
]
},
{
"values": [
"204352380837189.050001"
]
},
{
"values": [
"204694947715562.000001"
]
},
{
"values": [
"205044958345323.020002"
]
},
{
"values": [
"205376653885567.000002"
]
},
{
"values": [
"205708194566217.040001"
]
},
{
"values": [
"206019246459845.000002"
]
},
{
"values": [
"206328628269661.020001"
]
},
{
"values": [
"206628221303161.000001"
]
},
{
"values": [
"206927716356124.050002"
]
},
{
"values": [
"207227120176009.090001"
]
},
{
"values": [
"207500675828738.040001"
]
},
{
"values": [
"207816256311462.000001"
]
},
{
"values": [
"208130807246103.000001"
]
},
{
"values": [
"208483520084669.020001"
]
},
{
"values": [
"208807125005906.020002"
]
},
{
"values": [
"209121412207743.020001"
]
},
{
"values": [
"209437311304962.010001"
]
},
{
"values": [
"209746953505883.010001"
]
},
{
"values": [
"210047063105221.010002"
]
},
{
"values": [
"210353029456519.000001"
]
},
{
"values": [
"210650263506847.000001"
]
},
{
"values": [
"210938957763803.000001"
]
},
{
"values": [
"211225800004018.000001"
]
},
{
"values": [
"211502130806494.000001"
]
},
{
"values": [
"211737573865694.000001"
]
},
{
"values": [
"211996233748044.010001"
]
},
{
"values": [
"212254253769320.010002"
]
},
{
"values": [
"212512512505708.000001"
]
},
{
"values": [
"212764916414839.010002"
]
},
{
"values": [
"213028782860657.000001"
]
},
{
"values": [
"213267278505776.020002"
]
}
]
},
"entry_names": {}
}
}
}
}
],
"task_settings": {
"source_settings": {},
"target_settings": {
"default_schema": "MyUser",
"truncate_table_if_exists": true,
"drop_table_if_exists": false,
"queue_settings": {
"message_shape": {},
"key_shape": {}
},
"ftm_settings": {},
"artifacts_cleanup_enabled": false,
"ddl_handling_policy": {}
},
"sorter_settings": {
"local_transactions_storage": {}
},
"common_settings": {
"lob_max_size": 32,
"change_table_settings": {
"handle_ddl": false,
"header_columns_settings": {}
},
"audit_table_settings": {},
"dr_settings": {},
"statistics_table_settings": {},
"bidi_table_settings": {},
"task_uuid": "ec152817-7915-41a9-b5e7-fc2c868eef27",
"status_table_settings": {},
"suspended_tables_table_settings": {},
"history_table_settings": {},
"exception_table_settings": {},
"recovery_table_settings": {},
"data_batching_settings": {},
"data_batching_table_settings": {},
"log_stream_settings_depricated": {},
"ddl_history_table_settings": {},
"customized_charset_settings": {
"validation": {
"sub_char": 0
}
}
}
},
"configurations": [
{
"name": "MyUser.myTable"
}
]
}
]
},
"_version": {
"version": "2025.5.0.134",
"version_major": 2025,
"version_minor": 5,
"version_revision": 134,
"fips": 0
},
"description": "hostname"
}

[john_wang]

Thank you so much for your outstanding support! @QuiosaEvaristo , and @Vegy