[ Nov 25 19:31:02 Enabled. ]
[ Nov 25 19:31:02 Rereading configuration. ]
[ Nov 25 19:31:05 Rereading configuration. ]
[ Nov 25 19:31:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.021543434Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.025032193Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.025043323Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.041398254Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.110202364Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.550002318Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:59099"}
{"msg":"region 2faecc4f-d361-463e-a36b-fcca4cab1ae8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.577710032Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.581999546Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.586658117Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"212e2b4d-7e36-4cd3-a26b-0c1252c397d1","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":12817,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.592939281Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.592985531Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2faecc4f-d361-463e-a36b-fcca4cab1ae8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.666644467Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"2faecc4f-d361-463e-a36b-fcca4cab1ae8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2faecc4f-d361-463e-a36b-fcca4cab1ae8\" \"--data\" \"/data/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.668647016Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"2faecc4f-d361-463e-a36b-fcca4cab1ae8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.813348675Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"5dd99464-1ab5-43e9-b5f6-cd66929e532b","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.832089843Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"2faecc4f-d361-463e-a36b-fcca4cab1ae8","component":"worker"}
{"msg":"region 2faecc4f-d361-463e-a36b-fcca4cab1ae8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.832148674Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.832376138Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.832771516Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.837472218Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.851951782Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.855320949Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.856102324Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.856165065Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.856201866Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.856245567Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.867677501Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.877747818Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.309202383Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"8fe65121-9dd9-41b7-a24c-2919f5fae64e","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.828851369Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","method":"GET","req_id":"b036ced5-8979-4b03-b764-0d8e65a01a49","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.831383289Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"dataset is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.841768052Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"region 2faecc4f-d361-463e-a36b-fcca4cab1ae8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.850607746Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.85083803Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.850907272Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","method":"DELETE","req_id":"d9d48b59-db9e-4eeb-bf4e-5c26d5d2d1af","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19538,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.853969332Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","method":"GET","req_id":"677180dc-6eec-4217-b4c2-86d3af421e68","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-2faecc4f-d361-463e-a36b-fcca4cab1ae8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.854057624Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.858588152Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.868712801Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"2faecc4f-d361-463e-a36b-fcca4cab1ae8","component":"worker"}
{"msg":"region 2faecc4f-d361-463e-a36b-fcca4cab1ae8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.946024887Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.946402885Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/2faecc4f-d361-463e-a36b-fcca4cab1ae8","method":"GET","req_id":"20c73011-cc99-445e-8984-df5446db72ea","remote_addr":"[fd00:1122:3344:101::b]:59099","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.579578693Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45993"}
{"msg":"region 77468181-6b58-4b9f-865e-0bf19db11585 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.579840628Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.580045562Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"ebd481b4-bd90-4b00-b3d2-d0420e3dab4c","remote_addr":"[fd00:1122:3344:101::a]:45993","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":246,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.580072552Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.587967457Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.588002538Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"77468181-6b58-4b9f-865e-0bf19db11585\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/77468181-6b58-4b9f-865e-0bf19db11585\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.63347184Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"77468181-6b58-4b9f-865e-0bf19db11585","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"77468181-6b58-4b9f-865e-0bf19db11585\" \"--data\" \"/data/regions/77468181-6b58-4b9f-865e-0bf19db11585\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.635406968Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"77468181-6b58-4b9f-865e-0bf19db11585","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.708267837Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"77468181-6b58-4b9f-865e-0bf19db11585","component":"worker"}
{"msg":"region 77468181-6b58-4b9f-865e-0bf19db11585 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.708305028Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.708422191Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.709838548Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.713321147Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.720824204Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.723636549Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.723974436Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.724043147Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.724087638Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.724125109Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.728548026Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.733257438Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.929519189Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"a6a146e7-9b69-4c3b-b6df-3f92e8e68064","remote_addr":"[fd00:1122:3344:101::a]:45993","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.635117338Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33578"}
{"msg":"region 031e9996-035d-4700-bd05-a1ffbf8a174a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.63525667Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.639435252Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.639455843Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"2fa8bbdc-4291-49fd-a4ca-389057da17f9","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":4230,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.653783314Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.653824855Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"031e9996-035d-4700-bd05-a1ffbf8a174a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/031e9996-035d-4700-bd05-a1ffbf8a174a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.710346824Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"031e9996-035d-4700-bd05-a1ffbf8a174a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"031e9996-035d-4700-bd05-a1ffbf8a174a\" \"--data\" \"/data/regions/031e9996-035d-4700-bd05-a1ffbf8a174a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.71222617Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"031e9996-035d-4700-bd05-a1ffbf8a174a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.792805962Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"region":"031e9996-035d-4700-bd05-a1ffbf8a174a","component":"worker"}
{"msg":"region 031e9996-035d-4700-bd05-a1ffbf8a174a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.792859043Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.792950234Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.794102487Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.803699685Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.817848373Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.821921323Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/031e9996-035d-4700-bd05-a1ffbf8a174a","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.822225909Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.82227817Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.822323491Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.822363302Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.826277108Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.832134003Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:29.999498026Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions","method":"POST","req_id":"464f0dae-17c7-4637-842a-ab944928c2b1","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":105,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.034253938Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/77468181-6b58-4b9f-865e-0bf19db11585","method":"GET","req_id":"ff7dd7e3-15e3-436d-8b0b-f078ae2bb3ca","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/77468181-6b58-4b9f-865e-0bf19db11585\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.037596194Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"dataset is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.047401586Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"snapshot is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.055504145Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.055548136Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.066297287Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/77468181-6b58-4b9f-865e-0bf19db11585/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294","method":"GET","req_id":"ea596fcd-2835-46ad-b305-3513ec8d1030","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28721,"response_code":"200"}
{"msg":"path is \"/data/regions/77468181-6b58-4b9f-865e-0bf19db11585\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.069589331Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"dataset is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.078397764Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"snapshot is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.087634906Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.087661856Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 77468181-6b58-4b9f-865e-0bf19db11585-a71d0d30-1bc4-4c84-887d-7dc2bb733294 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.096097452Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"applying SMF actions for region 77468181-6b58-4b9f-865e-0bf19db11585 running snapshot a71d0d30-1bc4-4c84-887d-7dc2bb733294 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.096283325Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.096310636Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/77468181-6b58-4b9f-865e-0bf19db11585/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294/run","method":"POST","req_id":"1d462de2-2958-4dd5-aa52-ba8b48165fb2","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26761,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-77468181-6b58-4b9f-865e-0bf19db11585-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.099010179Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-77468181-6b58-4b9f-865e-0bf19db11585-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.102328024Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-77468181-6b58-4b9f-865e-0bf19db11585-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.114434091Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/77468181-6b58-4b9f-865e-0bf19db11585/.zfs/snapshot/a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.114703626Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.114759318Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.114807479Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.114844879Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.11489702Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.118595993Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.123673453Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"worker"}
{"msg":"region 77468181-6b58-4b9f-865e-0bf19db11585 running snapshot a71d0d30-1bc4-4c84-887d-7dc2bb733294 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.123683513Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"path is \"/data/regions/77468181-6b58-4b9f-865e-0bf19db11585\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.266688648Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"dataset is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.275531322Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"datafile"}
{"msg":"snapshot is oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.283629381Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_df57d550-ef00-4438-9a9e-b1b08bdaa525/crucible/regions/77468181-6b58-4b9f-865e-0bf19db11585\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.283662192Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.291955894Z","hostname":"oxz_crucible_fa3bd66a-1a51-4937-9188-36e5fc334f7c","pid":28508,"uri":"/crucible/0/regions/77468181-6b58-4b9f-865e-0bf19db11585/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294/run","method":"POST","req_id":"c21f4b3d-0a2f-4af6-a460-cbc9f2e3c2df","remote_addr":"[fd00:1122:3344:101::a]:33578","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25767,"response_code":"200"}