[ Oct 28 02:47:37 Enabled. ]
[ Oct 28 02:47:37 Rereading configuration. ]
[ Oct 28 02:47:42 Rereading configuration. ]
[ Oct 28 02:47:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 75ea3408643664877aa603ffc65996ed6dc34c0a\nCommit timestamp: 2025-10-24T15:21:57.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.79996517Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473}
{"msg":"dataset: \"oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.800940402Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.800953302Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.800959043Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.81555462Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.896441756Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.087826884Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49771"}
{"msg":"region b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.124086801Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.126863512Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.129486487Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"ea717f2f-2f1f-4f81-b239-ff7d95c45472","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19265,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.135897297Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.135914917Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.235267657Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\" \"--data\" \"/data/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.23535127Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.367139791Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"78933ebe-dfff-41cd-962f-6c431c67478d","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.512111773Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","component":"worker"}
{"msg":"region b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.512189015Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.512367281Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.512749854Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.518311275Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.538848087Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.541790113Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.542486676Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.542558509Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.54260216Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.542639311Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.546864849Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:12.552196424Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.084801984Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"2e2d7e90-9893-4ed8-a6d3-f3f34cf20a90","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.483879998Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","method":"GET","req_id":"469e0f6f-71b9-494f-8417-4752610b68c7","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":95,"response_code":200}
{"msg":"path is \"/data/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.488649574Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"dataset is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.497827204Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"region b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.505772914Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.505876797Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","method":"DELETE","req_id":"22ea857f-6a7a-42cf-b86f-8df470a7da90","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17255,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.505902878Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.506797887Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.509118143Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","method":"GET","req_id":"e7eb9029-d5f1-4530-9aad-3f3fbbfdb766","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.511094698Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.519771961Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","component":"worker"}
{"msg":"region b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.580701325Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:13.649481114Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/b74e11d5-c94b-4a0f-a14a-c7ed7bb467eb","method":"GET","req_id":"860dc8f0-6016-45a1-b19c-18c97a697b5f","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region 03fd109c-90a6-4781-8dcc-6bd3809710bb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.709520666Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.709661261Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"313505eb-be7b-4872-97c9-5a79aa6a2823","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":201,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.709686292Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.715513033Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.715536133Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"03fd109c-90a6-4781-8dcc-6bd3809710bb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.758675984Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"03fd109c-90a6-4781-8dcc-6bd3809710bb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"03fd109c-90a6-4781-8dcc-6bd3809710bb\" \"--data\" \"/data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.758718495Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"03fd109c-90a6-4781-8dcc-6bd3809710bb","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.836097227Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"03fd109c-90a6-4781-8dcc-6bd3809710bb","component":"worker"}
{"msg":"region 03fd109c-90a6-4781-8dcc-6bd3809710bb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.836132188Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.836237471Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.837656538Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.840930225Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.847974665Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.852352638Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.852637608Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.852689939Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.852732341Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.852774062Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.855923265Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.860576037Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.893526725Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"9f812784-111c-4a13-bdf0-7c667f77c2a5","remote_addr":"[fd00:1122:3344:101::a]:49771","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.664323938Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33693"}
{"msg":"region 86b0bf69-9f17-4abe-8829-d47de64024e7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.665161296Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.665385443Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.665394553Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"a6c7bb06-ee92-4c5f-b920-2ef0e7c3f8c4","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":1020,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.67474875Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.67477898Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"86b0bf69-9f17-4abe-8829-d47de64024e7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/86b0bf69-9f17-4abe-8829-d47de64024e7\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.721448057Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"86b0bf69-9f17-4abe-8829-d47de64024e7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"86b0bf69-9f17-4abe-8829-d47de64024e7\" \"--data\" \"/data/regions/86b0bf69-9f17-4abe-8829-d47de64024e7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.721491268Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"86b0bf69-9f17-4abe-8829-d47de64024e7","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.798332331Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"region":"86b0bf69-9f17-4abe-8829-d47de64024e7","component":"worker"}
{"msg":"region 86b0bf69-9f17-4abe-8829-d47de64024e7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.798362492Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.798473565Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.815224673Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.827270298Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.833988637Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions","method":"POST","req_id":"919af6e4-13b3-4524-a985-650d518f7ae6","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.836923173Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.840006004Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/86b0bf69-9f17-4abe-8829-d47de64024e7","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.840341615Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.840389457Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.840437808Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.8404917Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.844987137Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:46.849630889Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.921782208Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","method":"GET","req_id":"7bf5757a-4d87-45af-a115-71e1bd630542","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"path is \"/data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.925352975Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"dataset is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.931779295Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"snapshot is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.938427903Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.938455894Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.946922141Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","method":"GET","req_id":"310833f3-e322-47a8-9e31-ddd841432fdc","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21593,"response_code":200}
{"msg":"path is \"/data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.950552589Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"dataset is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.956169163Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"snapshot is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.96309741Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.963148721Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 03fd109c-90a6-4781-8dcc-6bd3809710bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.969955834Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.970110899Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a/run","method":"POST","req_id":"1585a761-f4c0-4a64-9e0f-26de75844a86","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19572,"response_code":200}
{"msg":"applying SMF actions for region 03fd109c-90a6-4781-8dcc-6bd3809710bb running snapshot 8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.97014286Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-03fd109c-90a6-4781-8dcc-6bd3809710bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.972758526Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-03fd109c-90a6-4781-8dcc-6bd3809710bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.975449364Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-03fd109c-90a6-4781-8dcc-6bd3809710bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.984857322Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb/.zfs/snapshot/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.985147131Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.985189232Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.985230234Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.985265785Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.985312046Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.988507311Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.992935206Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"worker"}
{"msg":"region 03fd109c-90a6-4781-8dcc-6bd3809710bb running snapshot 8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:47.992942316Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"path is \"/data/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.119445924Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"dataset is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.125845583Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"datafile"}
{"msg":"snapshot is oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.132626965Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_355310cf-0a0c-4388-9855-4febb8f0fe6e/crucible/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.132672737Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.138406704Z","hostname":"oxz_crucible_d72b6520-a49b-4545-a0ca-c3710301f026","pid":24473,"uri":"/crucible/0/regions/03fd109c-90a6-4781-8dcc-6bd3809710bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a/run","method":"POST","req_id":"5ef575c0-66f3-484b-80b6-a4b6d7826a95","remote_addr":"[fd00:1122:3344:101::a]:33693","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19011,"response_code":200}