[ Oct 18 05:35:49 Enabled. ]
[ Oct 18 05:35:49 Rereading configuration. ]
[ Oct 18 05:35:51 Rereading configuration. ]
[ Oct 18 05:35:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:56.485927616Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:56.487361465Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:56.487370285Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:56.501966127Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:35:56.787951439Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.514496113Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64857"}
{"msg":"region 2e943a75-1273-4000-a655-58ade801e872 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.537567314Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.542063903Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions","method":"POST","req_id":"d7af040f-faa6-4655-ac8b-908022f2c3d7","remote_addr":"[fd00:1122:3344:101::a]:64857","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":10452,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.543457791Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.552358019Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.55239265Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2e943a75-1273-4000-a655-58ade801e872\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2e943a75-1273-4000-a655-58ade801e872\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.61848361Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"2e943a75-1273-4000-a655-58ade801e872","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2e943a75-1273-4000-a655-58ade801e872\" \"--data\" \"/data/regions/2e943a75-1273-4000-a655-58ade801e872\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.622620833Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"2e943a75-1273-4000-a655-58ade801e872","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.755748462Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"2e943a75-1273-4000-a655-58ade801e872","component":"worker"}
{"msg":"region 2e943a75-1273-4000-a655-58ade801e872 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.755803823Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.755881544Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.75817553Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.762301723Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.780280632Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.783870903Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.789346703Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.789414324Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.789462415Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.789505946Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.789787162Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions","method":"POST","req_id":"3b02fe66-0355-4c72-808f-957a480fd367","remote_addr":"[fd00:1122:3344:101::a]:64857","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.794838702Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:37:10.801130338Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.437387006Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45765"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.440712023Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/2e943a75-1273-4000-a655-58ade801e872","method":"GET","req_id":"8192feab-e74a-448a-b11f-adc9263e90df","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":3277,"response_code":"200"}
{"msg":"path is \"/data/regions/2e943a75-1273-4000-a655-58ade801e872\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.44407612Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"dataset is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.452292544Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"snapshot is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.46160036Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.461626591Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.475030818Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/2e943a75-1273-4000-a655-58ade801e872/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112","method":"GET","req_id":"6d130651-40c1-4258-b82c-fa47500c0628","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":30964,"response_code":"200"}
{"msg":"path is \"/data/regions/2e943a75-1273-4000-a655-58ade801e872\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.478259263Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"dataset is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.486419326Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"snapshot is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.49414159Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.494176761Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2e943a75-1273-4000-a655-58ade801e872-cf4614e9-3156-42ca-be3e-103ee4c58112 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.501674541Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"applying SMF actions for region 2e943a75-1273-4000-a655-58ade801e872 running snapshot cf4614e9-3156-42ca-be3e-103ee4c58112 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.502263653Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.502278713Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/2e943a75-1273-4000-a655-58ade801e872/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112/run","method":"POST","req_id":"109ad078-0096-4516-b20a-667b6b118d5a","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24058,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.51064815Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.510754592Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.510856694Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.510864384Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.510918006Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.510981787Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.511093769Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.511169031Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.511227332Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.511285183Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.51816886Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2e943a75-1273-4000-a655-58ade801e872-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.518629989Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2e943a75-1273-4000-a655-58ade801e872-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.521250102Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2e943a75-1273-4000-a655-58ade801e872-cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.531804543Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e943a75-1273-4000-a655-58ade801e872/.zfs/snapshot/cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.532122029Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.53217499Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.532216851Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.532254962Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.532304863Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.535867414Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.540222351Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"region 2e943a75-1273-4000-a655-58ade801e872 running snapshot cf4614e9-3156-42ca-be3e-103ee4c58112 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.540239561Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"path is \"/data/regions/2e943a75-1273-4000-a655-58ade801e872\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.635874462Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"dataset is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.643671387Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"snapshot is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872@cf4614e9-3156-42ca-be3e-103ee4c58112","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.652198008Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/2e943a75-1273-4000-a655-58ade801e872\", \"cf4614e9-3156-42ca-be3e-103ee4c58112\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.652223088Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:39.660520464Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/2e943a75-1273-4000-a655-58ade801e872/snapshots/cf4614e9-3156-42ca-be3e-103ee4c58112/run","method":"POST","req_id":"06fbaba0-7c65-41e9-a821-9f44ca4066e9","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25430,"response_code":"200"}
{"msg":"region 1897462e-d16d-4e19-aa77-6e62106d98c8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.863527414Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.863715998Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions","method":"POST","req_id":"7f25184f-5740-4894-bc57-31d760085196","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":263,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.863745798Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.873061674Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.873094035Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1897462e-d16d-4e19-aa77-6e62106d98c8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1897462e-d16d-4e19-aa77-6e62106d98c8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.939310798Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"1897462e-d16d-4e19-aa77-6e62106d98c8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1897462e-d16d-4e19-aa77-6e62106d98c8\" \"--data\" \"/data/regions/1897462e-d16d-4e19-aa77-6e62106d98c8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:42.939361549Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"1897462e-d16d-4e19-aa77-6e62106d98c8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.013844247Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"1897462e-d16d-4e19-aa77-6e62106d98c8","component":"worker"}
{"msg":"region 1897462e-d16d-4e19-aa77-6e62106d98c8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.013899028Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.01401361Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.01451724Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.019971169Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.027117742Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.030298415Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.030568991Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.030640372Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.030697133Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.030766645Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.034976929Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039099231Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039216813Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039309935Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039317315Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039358646Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.039406447Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.04354901Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.043606081Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.043653352Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.043699733Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.047150812Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.049364026Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:40:43.127026478Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions","method":"POST","req_id":"16341d0f-1c10-4a02-9bfe-18a6488d78c4","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.598757261Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","method":"GET","req_id":"a4653b42-9fdf-4eb0-9376-4b0ac9926904","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"path is \"/data/regions/1897462e-d16d-4e19-aa77-6e62106d98c8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.602398404Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"dataset is oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.615332142Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"region 1897462e-d16d-4e19-aa77-6e62106d98c8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.625088367Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.625333062Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-1897462e-d16d-4e19-aa77-6e62106d98c8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.626395443Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.628204799Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","method":"DELETE","req_id":"89e5565a-fd62-40bf-9201-4af7d5421628","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25832,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631440153Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","method":"GET","req_id":"5b92b9c3-15ba-44aa-a5f2-ee2a7241c1ab","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":795,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631616437Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631714249Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631801431Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631855872Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631863572Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.631920093Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e943a75-1273-4000-a655-58ade801e872","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.63227911Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.632347462Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.632423923Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.632477324Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.636692369Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.638066916Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3d9295fb-ac56-4471-adff-db2072dd16c5/crucible/regions/1897462e-d16d-4e19-aa77-6e62106d98c8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.648885562Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"region":"1897462e-d16d-4e19-aa77-6e62106d98c8","component":"worker"}
{"msg":"region 1897462e-d16d-4e19-aa77-6e62106d98c8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.702993053Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T05:41:09.737255757Z","hostname":"oxz_crucible_a30d59da-d5c1-4812-abc8-ad4677958471","pid":22701,"uri":"/crucible/0/regions/1897462e-d16d-4e19-aa77-6e62106d98c8","method":"GET","req_id":"66671629-6bf1-4753-8cbe-6f525a98b9b8","remote_addr":"[fd00:1122:3344:101::a]:45765","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":80,"response_code":"200"}