[ Mar 18 18:17:19 Enabled. ]
[ Mar 18 18:17:19 Rereading configuration. ]
[ Mar 18 18:17:22 Rereading configuration. ]
[ Mar 18 18:17:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.39490256Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542}
{"msg":"dataset: \"oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.395656131Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.395666191Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.395671491Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.406595792Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.528250252Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.495863648Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41588"}
{"msg":"region 734e05d2-c7e0-417b-8e1e-d15f9df47221 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.522181852Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.524963069Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.52500888Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"f7971dd2-8b9c-4db9-85f1-957509ce4796","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15186,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.535472329Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.535488039Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"734e05d2-c7e0-417b-8e1e-d15f9df47221\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.608754967Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"734e05d2-c7e0-417b-8e1e-d15f9df47221","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"734e05d2-c7e0-417b-8e1e-d15f9df47221\" \"--data\" \"/data/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.615749509Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"734e05d2-c7e0-417b-8e1e-d15f9df47221","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.660735638Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"581f9f8e-36c9-419b-86af-ebe904468fce","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.955985598Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"734e05d2-c7e0-417b-8e1e-d15f9df47221","component":"worker"}
{"msg":"region 734e05d2-c7e0-417b-8e1e-d15f9df47221 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.95606093Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.956250305Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.956587105Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.962007794Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.981309005Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.984521724Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.985224803Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.985286615Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.985328946Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.985364547Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.991352742Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:52.997045619Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.350163333Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"0c6d7416-b48a-4568-9344-645d7f5dc5c4","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.733959552Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","method":"GET","req_id":"7f02a117-2524-4935-8fbd-337fcaa20171","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"path is \"/data/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.737022777Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"dataset is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.745030067Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"region 734e05d2-c7e0-417b-8e1e-d15f9df47221 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.752047971Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.752212905Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","method":"DELETE","req_id":"fa21f76d-cc4a-4206-a9d8-466c2a09fbf2","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15207,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.752257046Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-734e05d2-c7e0-417b-8e1e-d15f9df47221 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.75311833Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.755382062Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","method":"GET","req_id":"eea4f45a-d133-4437-837e-2fe857732d7c","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":30,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.76690774Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.776633908Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"734e05d2-c7e0-417b-8e1e-d15f9df47221","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.836365742Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","method":"GET","req_id":"050f1de6-0573-4678-b235-f5ec32f3ba95","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region 734e05d2-c7e0-417b-8e1e-d15f9df47221 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.850977075Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:53.945781406Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/734e05d2-c7e0-417b-8e1e-d15f9df47221","method":"GET","req_id":"90006197-f8d1-4b98-abd5-12e5ec8b162f","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":39,"response_code":200}
{"msg":"region 3dd3e1c7-f185-4d77-b741-f364d53e3f00 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.25513775Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.255393527Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"8c913688-1ad3-4caa-b19c-ccc48ef10b36","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":307,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.255458559Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.262287677Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.262346728Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3dd3e1c7-f185-4d77-b741-f364d53e3f00\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.334932977Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"3dd3e1c7-f185-4d77-b741-f364d53e3f00","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3dd3e1c7-f185-4d77-b741-f364d53e3f00\" \"--data\" \"/data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.334997449Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"3dd3e1c7-f185-4d77-b741-f364d53e3f00","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.445647336Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"3dd3e1c7-f185-4d77-b741-f364d53e3f00","component":"worker"}
{"msg":"region 3dd3e1c7-f185-4d77-b741-f364d53e3f00 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.445686737Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.445886193Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.447033714Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.450267343Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.457515243Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.460354621Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.460636879Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.460690321Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.460732062Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.460767472Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.465113592Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.469884414Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:56.544272462Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"8ed369b1-b194-48a0-a036-de17167ff9c3","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.260370551Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44677"}
{"msg":"region b0afccc3-76b3-4e2c-9347-e22923b93a4c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.261197463Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.26143132Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"1a673581-f67a-463c-9833-384aa9916e47","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":1019,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.26145741Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.268269948Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.268291959Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b0afccc3-76b3-4e2c-9347-e22923b93a4c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b0afccc3-76b3-4e2c-9347-e22923b93a4c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.319036146Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"b0afccc3-76b3-4e2c-9347-e22923b93a4c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b0afccc3-76b3-4e2c-9347-e22923b93a4c\" \"--data\" \"/data/regions/b0afccc3-76b3-4e2c-9347-e22923b93a4c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.319093017Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"b0afccc3-76b3-4e2c-9347-e22923b93a4c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.396948701Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"region":"b0afccc3-76b3-4e2c-9347-e22923b93a4c","component":"worker"}
{"msg":"region b0afccc3-76b3-4e2c-9347-e22923b93a4c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.396980092Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.397107226Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.415237895Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.423833502Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions","method":"POST","req_id":"336ba08a-0b10-4d24-8b1a-547ed0c717b7","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.426802074Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.439087492Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.443672818Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b0afccc3-76b3-4e2c-9347-e22923b93a4c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.443984097Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.444034508Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.444075049Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.44411297Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.447854193Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:02.452859821Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.94352967Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","method":"GET","req_id":"288aa73c-5ead-4425-8093-c803c175a91e","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"path is \"/data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.946574994Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"dataset is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.952693382Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"snapshot is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00@29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.958697118Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\", \"29255665-7738-4dbd-8694-8c193a17758a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.958722879Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.966853732Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00/snapshots/29255665-7738-4dbd-8694-8c193a17758a","method":"GET","req_id":"df871fb9-190a-4251-a6dc-68c76efe43dd","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20305,"response_code":200}
{"msg":"path is \"/data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.970683388Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"dataset is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.976092467Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"snapshot is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00@29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.982453552Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\", \"29255665-7738-4dbd-8694-8c193a17758a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.982472962Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 3dd3e1c7-f185-4d77-b741-f364d53e3f00-29255665-7738-4dbd-8694-8c193a17758a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.989508016Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.98964133Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00/snapshots/29255665-7738-4dbd-8694-8c193a17758a/run","method":"POST","req_id":"e5767d3a-6999-4fef-93fa-d8fb04c793aa","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18974,"response_code":200}
{"msg":"applying SMF actions for region 3dd3e1c7-f185-4d77-b741-f364d53e3f00 running snapshot 29255665-7738-4dbd-8694-8c193a17758a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.989658871Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-3dd3e1c7-f185-4d77-b741-f364d53e3f00-29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:03.994575266Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3dd3e1c7-f185-4d77-b741-f364d53e3f00-29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.002452223Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3dd3e1c7-f185-4d77-b741-f364d53e3f00-29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.011856932Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00/.zfs/snapshot/29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.012158Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.012200821Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.012243642Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.012279273Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.012322895Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.015761969Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.020452019Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"worker"}
{"msg":"region 3dd3e1c7-f185-4d77-b741-f364d53e3f00 running snapshot 29255665-7738-4dbd-8694-8c193a17758a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.020464599Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"path is \"/data/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.149884353Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"dataset is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.155900128Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"datafile"}
{"msg":"snapshot is oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00@29255665-7738-4dbd-8694-8c193a17758a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.162080788Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32cb896f-f5d9-4f6e-8da9-030aba90df9a/crucible/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00\", \"29255665-7738-4dbd-8694-8c193a17758a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.162107979Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:04.168281609Z","hostname":"oxz_crucible_a633b0db-a658-4eda-959f-328e3aa4a8e2","pid":24542,"uri":"/crucible/0/regions/3dd3e1c7-f185-4d77-b741-f364d53e3f00/snapshots/29255665-7738-4dbd-8694-8c193a17758a/run","method":"POST","req_id":"44cce6f2-7ac7-4e0f-b83c-baa4e81b4156","remote_addr":"[fd00:1122:3344:101::a]:44677","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18432,"response_code":200}