[ Mar 17 21:02:05 Enabled. ]
[ Mar 17 21:02:05 Rereading configuration. ]
[ Mar 17 21:02:08 Rereading configuration. ]
[ Mar 17 21:02:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.272004132Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424}
{"msg":"dataset: \"oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.272785914Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.272802555Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.272808685Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.284767106Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:18.55146742Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.056861265Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38999"}
{"msg":"region 25f3f4f5-142a-4809-8e1c-333a3bba94ce state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.082326532Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.085679536Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.085732677Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"4bffab5f-6f10-492f-87c5-768bd8922909","remote_addr":"[fd00:1122:3344:101::b]:38999","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14273,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.096187844Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.096206724Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"25f3f4f5-142a-4809-8e1c-333a3bba94ce\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.176042761Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"25f3f4f5-142a-4809-8e1c-333a3bba94ce","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"25f3f4f5-142a-4809-8e1c-333a3bba94ce\" \"--data\" \"/data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.178404688Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"25f3f4f5-142a-4809-8e1c-333a3bba94ce","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.46628214Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"7a825672-77f9-496e-9e1a-f5a81e523040","remote_addr":"[fd00:1122:3344:101::b]:38999","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.624572321Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"25f3f4f5-142a-4809-8e1c-333a3bba94ce","component":"worker"}
{"msg":"region 25f3f4f5-142a-4809-8e1c-333a3bba94ce state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.624649382Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.624853636Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.625193031Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.631839697Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.647299124Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.651564812Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.652247134Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.652305435Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.652398236Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.652444777Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.659020122Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.664926186Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:02:59.731045783Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"c6f79185-0885-4d81-9358-0eaa5a21643e","remote_addr":"[fd00:1122:3344:101::b]:38999","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.623613668Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:63727"}
{"msg":"region ae1af704-3c1f-4f11-9b6a-74a2d35b40ae state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.624855298Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.624885849Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.624891909Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"59e121f4-4ddc-441f-857e-47b024b06601","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1110,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.631326052Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.631360722Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ae1af704-3c1f-4f11-9b6a-74a2d35b40ae\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.743385643Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ae1af704-3c1f-4f11-9b6a-74a2d35b40ae\" \"--data\" \"/data/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.743430434Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:28.977803421Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"38fb7491-39a5-47a1-9446-925f562c977c","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.026413988Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","component":"worker"}
{"msg":"region ae1af704-3c1f-4f11-9b6a-74a2d35b40ae state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.026443509Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.02654565Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.047330752Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.061099243Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.080278709Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.086826104Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.087142559Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.08718491Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.08722686Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.087259641Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.098237506Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.113629512Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.573085398Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49573"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.573268571Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/ae1af704-3c1f-4f11-9b6a-74a2d35b40ae","method":"GET","req_id":"f5c98b64-1e4b-4466-8053-bbba5646931b","remote_addr":"[fd00:1122:3344:101::b]:49573","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":86,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:29.715598066Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"f1736210-e947-4bd8-80ba-b653250736c2","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.689606327Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","method":"GET","req_id":"da705a3b-9a43-4a43-af28-04622d99eae9","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.693371708Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"dataset is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.701098891Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"snapshot is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce@6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.707533854Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\", \"6965c5a7-4b95-451c-a31c-bfdf67f893c4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.707560445Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.716498068Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce/snapshots/6965c5a7-4b95-451c-a31c-bfdf67f893c4","method":"GET","req_id":"522c98a4-546f-4003-b0e0-0eba7e3b2604","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23158,"response_code":200}
{"msg":"path is \"/data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.720570943Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"dataset is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.726985025Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"snapshot is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce@6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.734394794Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\", \"6965c5a7-4b95-451c-a31c-bfdf67f893c4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.734421914Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 25f3f4f5-142a-4809-8e1c-333a3bba94ce-6965c5a7-4b95-451c-a31c-bfdf67f893c4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.743865985Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"applying SMF actions for region 25f3f4f5-142a-4809-8e1c-333a3bba94ce running snapshot 6965c5a7-4b95-451c-a31c-bfdf67f893c4 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.743998137Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.744013837Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce/snapshots/6965c5a7-4b95-451c-a31c-bfdf67f893c4/run","method":"POST","req_id":"2c9439fb-d57b-4b7c-a571-d3be5373eb4c","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23459,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-25f3f4f5-142a-4809-8e1c-333a3bba94ce-6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.749129139Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-25f3f4f5-142a-4809-8e1c-333a3bba94ce-6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.772567064Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-25f3f4f5-142a-4809-8e1c-333a3bba94ce-6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.794519015Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce/.zfs/snapshot/6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.79484364Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.794897381Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.794929482Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.794970962Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.795026853Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.808964386Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.822420591Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"region 25f3f4f5-142a-4809-8e1c-333a3bba94ce running snapshot 6965c5a7-4b95-451c-a31c-bfdf67f893c4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.822446091Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"path is \"/data/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.980790383Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"dataset is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.987667873Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"snapshot is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce@6965c5a7-4b95-451c-a31c-bfdf67f893c4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.994734086Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce\", \"6965c5a7-4b95-451c-a31c-bfdf67f893c4\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:30.994752536Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:31.000989316Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/25f3f4f5-142a-4809-8e1c-333a3bba94ce/snapshots/6965c5a7-4b95-451c-a31c-bfdf67f893c4/run","method":"POST","req_id":"c9073714-5224-444c-b62b-1aed86f70c56","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20243,"response_code":200}
{"msg":"region 96327489-0902-458e-8e27-696ff120293d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.408428904Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.408616997Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.408646837Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"c54a9142-47ed-4853-90c1-618647ad286f","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":243,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.416748077Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.416766647Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"96327489-0902-458e-8e27-696ff120293d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/96327489-0902-458e-8e27-696ff120293d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.4800991Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"96327489-0902-458e-8e27-696ff120293d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"96327489-0902-458e-8e27-696ff120293d\" \"--data\" \"/data/regions/96327489-0902-458e-8e27-696ff120293d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.480159911Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"96327489-0902-458e-8e27-696ff120293d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.574519309Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"96327489-0902-458e-8e27-696ff120293d","component":"worker"}
{"msg":"region 96327489-0902-458e-8e27-696ff120293d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.57456638Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.574656831Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.575696108Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.579207894Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.587581238Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.591597392Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.591897237Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.591941378Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.591983499Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.592019179Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.595465994Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.606893387Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:13:36.713162446Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions","method":"POST","req_id":"f81f16e4-1f4c-40ec-9dff-d9f2cb5df46a","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.31434523Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/96327489-0902-458e-8e27-696ff120293d","method":"GET","req_id":"cd16ef55-c80f-4955-9fd1-7793236ac082","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"path is \"/data/regions/96327489-0902-458e-8e27-696ff120293d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.317997099Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"dataset is oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/96327489-0902-458e-8e27-696ff120293d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.329750176Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"region 96327489-0902-458e-8e27-696ff120293d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.336349002Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.336530715Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.336552275Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/96327489-0902-458e-8e27-696ff120293d","method":"DELETE","req_id":"29913d7b-8c99-42d0-a9ef-13e8e0060a31","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18562,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-96327489-0902-458e-8e27-696ff120293d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.337138085Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.339485292Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/96327489-0902-458e-8e27-696ff120293d","method":"GET","req_id":"32efbdb4-6350-4fd2-b498-c4f76ccde133","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":44,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.34937135Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8df4ff9f-dc53-43ab-9fe2-057d66770894/crucible/regions/96327489-0902-458e-8e27-696ff120293d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.35752036Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"region":"96327489-0902-458e-8e27-696ff120293d","component":"worker"}
{"msg":"region 96327489-0902-458e-8e27-696ff120293d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.412271556Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T21:14:02.441532183Z","hostname":"oxz_crucible_1ed1b1fa-9af9-4e31-b467-cb6634f907ce","pid":29424,"uri":"/crucible/0/regions/96327489-0902-458e-8e27-696ff120293d","method":"GET","req_id":"fc6b1082-c756-4c89-8ca3-dc245c32f309","remote_addr":"[fd00:1122:3344:101::b]:63727","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":46,"response_code":200}