[ Jul 30 01:30:26 Enabled. ]
[ Jul 30 01:30:26 Rereading configuration. ]
[ Jul 30 01:30:27 Rereading configuration. ]
[ Jul 30 01:30:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.373647628Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.375130813Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.375142623Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.416885237Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:30:29.515396276Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.59094624Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54734"}
{"msg":"region 666741d8-40c8-4e26-86bf-3f5682e66bf4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.611807352Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.616752562Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.619998765Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"712e9674-759c-4f1f-914a-da74ca733fcc","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14927,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.627727983Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.627746864Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"666741d8-40c8-4e26-86bf-3f5682e66bf4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.733372235Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"666741d8-40c8-4e26-86bf-3f5682e66bf4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"666741d8-40c8-4e26-86bf-3f5682e66bf4\" \"--data\" \"/data/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.735465046Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"666741d8-40c8-4e26-86bf-3f5682e66bf4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.930175631Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"666741d8-40c8-4e26-86bf-3f5682e66bf4","component":"worker"}
{"msg":"region 666741d8-40c8-4e26-86bf-3f5682e66bf4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.930228541Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.930313002Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.930706976Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.957858421Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.972615211Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.975579781Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.977712633Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"c9a395b4-240c-431b-bfa0-b1dd896c1d70","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":264,"response_code":"200"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.984685004Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.986929616Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.987008677Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.987082708Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.991613774Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:43.996951018Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.143831091Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","method":"GET","req_id":"42adff0b-9046-48f0-b9e9-e8f66029c4fa","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.14573231Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"dataset is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.155316207Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"region 666741d8-40c8-4e26-86bf-3f5682e66bf4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.16349786Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.163633691Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.163709672Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","method":"DELETE","req_id":"56071eaa-e60f-4fb2-a7d3-2ceca13d3799","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18006,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-666741d8-40c8-4e26-86bf-3f5682e66bf4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.164697732Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.167057226Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","method":"GET","req_id":"8731f853-f8f7-4a1e-bb0c-0c262d4d9dff","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.16838626Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.178448082Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"666741d8-40c8-4e26-86bf-3f5682e66bf4","component":"worker"}
{"msg":"region 666741d8-40c8-4e26-86bf-3f5682e66bf4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.234307078Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:45.263662416Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/666741d8-40c8-4e26-86bf-3f5682e66bf4","method":"GET","req_id":"b8102cd4-0002-4261-8c38-bb7037cc724a","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":497,"response_code":"200"}
{"msg":"region 6f247614-4250-4c35-9213-fbe29f96b4b7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.463497436Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.463631457Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.463646248Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"66304cb7-ac1c-4f09-808c-28e5fb30a8c3","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":782,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.473010563Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.473057483Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6f247614-4250-4c35-9213-fbe29f96b4b7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.518012679Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"6f247614-4250-4c35-9213-fbe29f96b4b7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6f247614-4250-4c35-9213-fbe29f96b4b7\" \"--data\" \"/data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.518080649Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"6f247614-4250-4c35-9213-fbe29f96b4b7","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.593880618Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"6f247614-4250-4c35-9213-fbe29f96b4b7","component":"worker"}
{"msg":"region 6f247614-4250-4c35-9213-fbe29f96b4b7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.593926669Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.59406222Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.595504535Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.59899589Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.606645228Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.609882561Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.610112353Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.610163994Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.610205864Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.610245824Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.613707099Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.61765771Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:31:46.760135795Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"239c4f0e-2bc8-4fc7-ba22-6dcd688e42a2","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.908935962Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60774"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.909100924Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","method":"GET","req_id":"cec0af1e-5154-469d-999d-863142e0f007","remote_addr":"[fd00:1122:3344:101::a]:60774","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.912340447Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"dataset is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.920660681Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"snapshot is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7@f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.929247148Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\", \"f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.929274929Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.94025577Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/6f247614-4250-4c35-9213-fbe29f96b4b7/snapshots/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","method":"GET","req_id":"b58bc5da-bb89-4196-9caa-3b1189b44d62","remote_addr":"[fd00:1122:3344:101::a]:60774","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31083,"response_code":"200"}
{"msg":"path is \"/data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.940419852Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"dataset is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.949437073Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"snapshot is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7@f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.957905919Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/6f247614-4250-4c35-9213-fbe29f96b4b7\", \"f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.957938059Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6f247614-4250-4c35-9213-fbe29f96b4b7-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.966992742Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions for region 6f247614-4250-4c35-9213-fbe29f96b4b7 running snapshot f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.967618708Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.967644788Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/6f247614-4250-4c35-9213-fbe29f96b4b7/snapshots/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b/run","method":"POST","req_id":"841a2369-3d2f-4a7c-a5c1-f2ae846d685f","remote_addr":"[fd00:1122:3344:101::a]:60774","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27222,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-6f247614-4250-4c35-9213-fbe29f96b4b7-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.974337436Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6f247614-4250-4c35-9213-fbe29f96b4b7-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.982376427Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6f247614-4250-4c35-9213-fbe29f96b4b7-f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996519301Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6f247614-4250-4c35-9213-fbe29f96b4b7/.zfs/snapshot/f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996750763Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996801694Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996848704Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996887535Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:06.996933045Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:07.001464871Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:07.006181229Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"region 6f247614-4250-4c35-9213-fbe29f96b4b7 running snapshot f5199ea3-dcee-4c01-98b8-ff5c9bf72a4b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:07.006204889Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.800732414Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56569"}
{"msg":"region c27bd160-968a-45de-a3e8-142fe074d377 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.80130083Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.801459471Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.801473151Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"402083b7-be7d-4970-859e-59709d451bd4","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":511,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.811949778Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.811976678Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c27bd160-968a-45de-a3e8-142fe074d377\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c27bd160-968a-45de-a3e8-142fe074d377\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.864664592Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"c27bd160-968a-45de-a3e8-142fe074d377","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c27bd160-968a-45de-a3e8-142fe074d377\" \"--data\" \"/data/regions/c27bd160-968a-45de-a3e8-142fe074d377\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.864714042Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"c27bd160-968a-45de-a3e8-142fe074d377","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.931918054Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"c27bd160-968a-45de-a3e8-142fe074d377","component":"worker"}
{"msg":"region c27bd160-968a-45de-a3e8-142fe074d377 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.931970935Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.932047636Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.933849364Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.9364716Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.944159728Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.947469542Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.947678914Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.947748445Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.947791775Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.947835705Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.951571848Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:09.956948748Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:10.047592497Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions","method":"POST","req_id":"313fb16e-8b2d-4497-ba60-58327ae6b38f","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":810,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.063726169Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/c27bd160-968a-45de-a3e8-142fe074d377","method":"GET","req_id":"47440f5e-dce9-4296-a06c-4f5b510b03f6","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/c27bd160-968a-45de-a3e8-142fe074d377\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.066128513Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"dataset is oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/c27bd160-968a-45de-a3e8-142fe074d377","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.077295576Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"region c27bd160-968a-45de-a3e8-142fe074d377 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.085968664Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.086175476Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.086238927Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/c27bd160-968a-45de-a3e8-142fe074d377","method":"DELETE","req_id":"78178b50-2462-49f9-8c32-87b381db5573","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20136,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c27bd160-968a-45de-a3e8-142fe074d377 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.087987975Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.08854206Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/c27bd160-968a-45de-a3e8-142fe074d377","method":"GET","req_id":"e16aca1f-47ba-4560-a10f-e3a01432aadb","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.093371369Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_33b09d10-f4a7-481b-a7f4-12388d3ec2fd/crucible/regions/c27bd160-968a-45de-a3e8-142fe074d377\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.102580783Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"region":"c27bd160-968a-45de-a3e8-142fe074d377","component":"worker"}
{"msg":"region c27bd160-968a-45de-a3e8-142fe074d377 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.150315327Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-30T01:37:34.177163959Z","hostname":"oxz_crucible_9a336735-2cd4-45ae-944c-a0b6166a0f02","pid":25395,"uri":"/crucible/0/regions/c27bd160-968a-45de-a3e8-142fe074d377","method":"GET","req_id":"b6e987dc-29f5-42f9-aea7-f4c9a54bd94c","remote_addr":"[fd00:1122:3344:101::a]:56569","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":"200"}