[ Mar 21 01:47:38 Enabled. ]
[ Mar 21 01:47:38 Rereading configuration. ]
[ Mar 21 01:47:41 Rereading configuration. ]
[ Mar 21 01:47:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.361671338Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481}
{"msg":"dataset: \"oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.362850901Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.362865311Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.362874221Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.380672255Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:47:48.54183775Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.494373041Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:63940"}
{"msg":"region 3cff7322-51e6-451b-84d5-583f0a6c97ff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.519627689Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.522539426Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.522576196Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions","method":"POST","req_id":"644914ee-7f9e-4218-b4b2-25a543a65079","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14090,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.531978108Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.532002318Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3cff7322-51e6-451b-84d5-583f0a6c97ff\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.586200166Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"3cff7322-51e6-451b-84d5-583f0a6c97ff","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3cff7322-51e6-451b-84d5-583f0a6c97ff\" \"--data\" \"/data/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.58851478Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"3cff7322-51e6-451b-84d5-583f0a6c97ff","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.829954786Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions","method":"POST","req_id":"82aa3e40-ce08-4e91-a6b1-baac22344bdb","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.939616405Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"3cff7322-51e6-451b-84d5-583f0a6c97ff","component":"worker"}
{"msg":"region 3cff7322-51e6-451b-84d5-583f0a6c97ff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.939665926Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.939758168Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.940097395Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.945638211Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.958900378Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.962233482Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.963042788Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.963101809Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.96314006Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.96317596Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.96783426Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:52.971993351Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.08684076Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions","method":"POST","req_id":"9ac47e6f-d573-4901-bf4a-5c9ef0ceb998","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.86257112Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","method":"GET","req_id":"694bd4a1-eb6a-4c93-9e9e-d2d5dbef373c","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"path is \"/data/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.866373433Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"dataset is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.874897568Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"region 3cff7322-51e6-451b-84d5-583f0a6c97ff state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.882555906Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.882703139Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.882727079Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","method":"DELETE","req_id":"f4519c1f-cdcf-4a62-b500-997101703959","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16336,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-3cff7322-51e6-451b-84d5-583f0a6c97ff (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.883871012Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.886630185Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","method":"GET","req_id":"f72948f3-3c8e-4c65-bb32-3e727a7016bc","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":37,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.889099542Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.897586557Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"3cff7322-51e6-451b-84d5-583f0a6c97ff","component":"worker"}
{"msg":"region 3cff7322-51e6-451b-84d5-583f0a6c97ff state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.971527355Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:53.99607441Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/3cff7322-51e6-451b-84d5-583f0a6c97ff","method":"GET","req_id":"e67dc742-ad59-42f8-95c4-52d04426104b","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"region 6aa9d8e9-da73-49aa-a86e-8314c67ad9f6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.906071798Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.906201641Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.906237542Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions","method":"POST","req_id":"9d68f83b-83a4-42f3-a99c-5b75f9e33347","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":180,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.912074874Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.912094425Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.962466828Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\" \"--data\" \"/data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:55.962499539Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.04534384Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"region":"6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","component":"worker"}
{"msg":"region 6aa9d8e9-da73-49aa-a86e-8314c67ad9f6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.04537662Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.045498003Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.046708856Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.050185053Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.057672548Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.061061093Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.061356039Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.06141275Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.061451441Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.061488932Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.065465429Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.070152439Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:48:56.216470736Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions","method":"POST","req_id":"f5d1c1e9-e66d-4ae0-b2af-e7c338d12792","remote_addr":"[fd00:1122:3344:101::b]:63940","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.780245577Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64905"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.781497681Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","method":"GET","req_id":"7c1565a3-933b-4c2f-8daa-fc3647b9f0bf","remote_addr":"[fd00:1122:3344:101::b]:64905","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":36,"response_code":200}
{"msg":"path is \"/data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.783559701Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"dataset is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.790486095Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"snapshot is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6@4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.797382218Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\", \"4e50dcf1-65f5-4a7f-bd57-7c06b443fd10\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.797405989Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.808013654Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6/snapshots/4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","method":"GET","req_id":"ad455794-fa34-4193-8710-76393323a649","remote_addr":"[fd00:1122:3344:101::b]:64905","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24476,"response_code":200}
{"msg":"path is \"/data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.811236626Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"dataset is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.816703781Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"snapshot is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6@4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.823711817Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\", \"4e50dcf1-65f5-4a7f-bd57-7c06b443fd10\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.823726317Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6aa9d8e9-da73-49aa-a86e-8314c67ad9f6-4e50dcf1-65f5-4a7f-bd57-7c06b443fd10 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.831317804Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"applying SMF actions for region 6aa9d8e9-da73-49aa-a86e-8314c67ad9f6 running snapshot 4e50dcf1-65f5-4a7f-bd57-7c06b443fd10 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.832220321Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.832237332Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6/snapshots/4e50dcf1-65f5-4a7f-bd57-7c06b443fd10/run","method":"POST","req_id":"d3f5c514-4069-4caa-86ac-e7f16bfa30c4","remote_addr":"[fd00:1122:3344:101::b]:64905","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20996,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6-4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.853221397Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6-4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.870650054Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6aa9d8e9-da73-49aa-a86e-8314c67ad9f6-4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.88391236Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6/.zfs/snapshot/4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.884106804Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.884146405Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.884184636Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.884231857Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.884272187Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.88801428Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.892583378Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"worker"}
{"msg":"region 6aa9d8e9-da73-49aa-a86e-8314c67ad9f6 running snapshot 4e50dcf1-65f5-4a7f-bd57-7c06b443fd10 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:21.892594918Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"path is \"/data/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:22.210097014Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"dataset is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:22.215983097Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"datafile"}
{"msg":"snapshot is oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6@4e50dcf1-65f5-4a7f-bd57-7c06b443fd10","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:22.221080186Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_efbc1277-5308-4b3a-8313-0a8ea441edaf/crucible/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6\", \"4e50dcf1-65f5-4a7f-bd57-7c06b443fd10\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:22.221102126Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-21T01:59:22.226015671Z","hostname":"oxz_crucible_e66f4189-2ef0-438d-9ec8-afabe60ebdff","pid":23481,"uri":"/crucible/0/regions/6aa9d8e9-da73-49aa-a86e-8314c67ad9f6/snapshots/4e50dcf1-65f5-4a7f-bd57-7c06b443fd10/run","method":"POST","req_id":"f0e79708-17b7-4fad-a192-9a42330e206a","remote_addr":"[fd00:1122:3344:101::b]:64905","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15946,"response_code":200}