[ Mar 26 05:46:14 Enabled. ]
[ Mar 26 05:46:14 Rereading configuration. ]
[ Mar 26 05:46:16 Rereading configuration. ]
[ Mar 26 05:46:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.090911731Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261}
{"msg":"dataset: \"oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.09128399Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.091297181Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.091306631Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.103794493Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:27.313338169Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.330283145Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39914"}
{"msg":"region 3e672d08-4c0d-441e-a9be-cbdc4db1a7a1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.363039232Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.366310587Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.366349007Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions","method":"POST","req_id":"6657db47-75ca-4110-8503-4217b8cc1b48","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17621,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.376475096Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.376495877Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.44883442Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\" \"--data\" \"/data/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.451262177Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.684569399Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions","method":"POST","req_id":"fb0b3583-193d-4bef-9fdd-d25221fe452f","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.798009191Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","component":"worker"}
{"msg":"region 3e672d08-4c0d-441e-a9be-cbdc4db1a7a1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.798066586Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.798327761Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.79867801Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.805738215Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.822414735Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.825482237Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.826145413Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.826202864Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.826252345Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.826289306Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.830019617Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:46:59.834116679Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.145367506Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions","method":"POST","req_id":"b282cfc2-e5de-44d3-8008-40b3a61673db","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.49101724Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","method":"GET","req_id":"aaed97a0-92fa-42f0-86ac-96c76eb559e6","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":81,"response_code":200}
{"msg":"path is \"/data/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.494371638Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"dataset is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.50300203Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"region 3e672d08-4c0d-441e-a9be-cbdc4db1a7a1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.511681204Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.511874148Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","method":"DELETE","req_id":"43dd4633-0e9a-42fe-a08b-335ecfa99471","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17530,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.511919569Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-3e672d08-4c0d-441e-a9be-cbdc4db1a7a1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.512857551Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.517041089Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","method":"GET","req_id":"792ddd3a-834d-4be9-b6c4-9c7ea1a5dea9","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.518464592Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.527048113Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","component":"worker"}
{"msg":"region 3e672d08-4c0d-441e-a9be-cbdc4db1a7a1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.631016818Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:00.637014369Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/3e672d08-4c0d-441e-a9be-cbdc4db1a7a1","method":"GET","req_id":"988bfb77-0bc3-4b94-a7d1-24131bd9c0a4","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"region 03853711-ca3c-45de-9c78-99b8b00fb2ce state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.603418512Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.60378082Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.603831632Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions","method":"POST","req_id":"8a81b15f-95cb-40ce-bf2e-0d1d908538ac","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":421,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.60971407Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.609732271Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"03853711-ca3c-45de-9c78-99b8b00fb2ce\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.667660566Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"03853711-ca3c-45de-9c78-99b8b00fb2ce","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"03853711-ca3c-45de-9c78-99b8b00fb2ce\" \"--data\" \"/data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.667737838Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"03853711-ca3c-45de-9c78-99b8b00fb2ce","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.741913915Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"region":"03853711-ca3c-45de-9c78-99b8b00fb2ce","component":"worker"}
{"msg":"region 03853711-ca3c-45de-9c78-99b8b00fb2ce state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.741939606Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.742167221Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.743538363Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.746374619Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.753594959Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.756565638Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.756852855Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.756906986Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.756915376Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.756959097Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.76091045Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.765236011Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:47:02.915410498Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions","method":"POST","req_id":"616b84a6-4bd0-47ed-b144-8c40c5fb58be","remote_addr":"[fd00:1122:3344:101::c]:39914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.462424959Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37534"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.463238368Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","method":"GET","req_id":"b6c06f74-2cad-42b2-b973-480074bd4299","remote_addr":"[fd00:1122:3344:101::c]:37534","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":38,"response_code":200}
{"msg":"path is \"/data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.465814588Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"dataset is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.472535226Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"snapshot is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce@225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.47912567Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\", \"225fa095-ca7c-4003-9052-9738cd2f5c1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.479161031Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.489433191Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce/snapshots/225fa095-ca7c-4003-9052-9738cd2f5c1e","method":"GET","req_id":"0ff89a6c-ae3f-4120-bed5-e0527be15880","remote_addr":"[fd00:1122:3344:101::c]:37534","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":23640,"response_code":200}
{"msg":"path is \"/data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.492417201Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"dataset is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.498498573Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"snapshot is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce@225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.505493777Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\", \"225fa095-ca7c-4003-9052-9738cd2f5c1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.505518258Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 03853711-ca3c-45de-9c78-99b8b00fb2ce-225fa095-ca7c-4003-9052-9738cd2f5c1e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.513691699Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.514520799Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce/snapshots/225fa095-ca7c-4003-9052-9738cd2f5c1e/run","method":"POST","req_id":"f25ec385-2951-47dd-8cb8-1b47f031f558","remote_addr":"[fd00:1122:3344:101::c]:37534","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22123,"response_code":200}
{"msg":"applying SMF actions for region 03853711-ca3c-45de-9c78-99b8b00fb2ce running snapshot 225fa095-ca7c-4003-9052-9738cd2f5c1e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.514537619Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-03853711-ca3c-45de-9c78-99b8b00fb2ce-225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.536454202Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-03853711-ca3c-45de-9c78-99b8b00fb2ce-225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.552243912Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-03853711-ca3c-45de-9c78-99b8b00fb2ce-225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566124537Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce/.zfs/snapshot/225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566376933Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566419483Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566463275Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566513226Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.566567377Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.570662903Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.57482702Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"worker"}
{"msg":"region 03853711-ca3c-45de-9c78-99b8b00fb2ce running snapshot 225fa095-ca7c-4003-9052-9738cd2f5c1e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.574837801Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"path is \"/data/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.757794114Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"dataset is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.764867879Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"datafile"}
{"msg":"snapshot is oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce@225fa095-ca7c-4003-9052-9738cd2f5c1e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.770741557Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_425e01eb-89e9-42ca-a214-11f61c7dde32/crucible/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce\", \"225fa095-ca7c-4003-9052-9738cd2f5c1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.770767127Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T05:57:44.777333121Z","hostname":"oxz_crucible_b5867ea1-1b08-4969-add4-dcd04d28fdf7","pid":26261,"uri":"/crucible/0/regions/03853711-ca3c-45de-9c78-99b8b00fb2ce/snapshots/225fa095-ca7c-4003-9052-9738cd2f5c1e/run","method":"POST","req_id":"e7776a5a-c3ed-4407-b917-4b43c9983e4b","remote_addr":"[fd00:1122:3344:101::c]:37534","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19564,"response_code":200}