[ Aug  5 16:46:20 Enabled. ]
[ Aug  5 16:46:20 Rereading configuration. ]
[ Aug  5 16:46:22 Rereading configuration. ]
[ Aug  5 16:46:28 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:28.534740578Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:28.535554551Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:28.535564441Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:28.547881571Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:28.932620162Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.354211123Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45969"}
{"msg":"region 2e9d3399-1c88-40d6-9437-68400e50f843 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.376846409Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.381755883Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.385902513Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions","method":"POST","req_id":"170aef72-06af-44b9-9266-7718a64db2cd","remote_addr":"[fd00:1122:3344:101::a]:45969","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":15880,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.393306519Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.39333258Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2e9d3399-1c88-40d6-9437-68400e50f843\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2e9d3399-1c88-40d6-9437-68400e50f843\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.458124795Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"2e9d3399-1c88-40d6-9437-68400e50f843","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2e9d3399-1c88-40d6-9437-68400e50f843\" \"--data\" \"/data/regions/2e9d3399-1c88-40d6-9437-68400e50f843\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.460153449Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"2e9d3399-1c88-40d6-9437-68400e50f843","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.584082792Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions","method":"POST","req_id":"4ada7a2a-44f3-45bb-bd5a-7b1230691ba4","remote_addr":"[fd00:1122:3344:101::a]:45969","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.603079755Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"2e9d3399-1c88-40d6-9437-68400e50f843","component":"worker"}
{"msg":"region 2e9d3399-1c88-40d6-9437-68400e50f843 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.603124915Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.603232945Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.603669912Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.609115609Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.620148817Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.623464224Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.632192222Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.634249478Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.634327269Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.63437557Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.638952318Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.643284242Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:47.020525475Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions","method":"POST","req_id":"c4266b28-8083-4681-a739-9a2f45cca06e","remote_addr":"[fd00:1122:3344:101::a]:45969","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.311801031Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60447"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.311906923Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/2e9d3399-1c88-40d6-9437-68400e50f843","method":"GET","req_id":"4c744753-3364-4f60-984e-dda455f610cc","remote_addr":"[fd00:1122:3344:101::a]:60447","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/2e9d3399-1c88-40d6-9437-68400e50f843\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.314515797Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"dataset is oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.323701524Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"snapshot is oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843@11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.333528282Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843\", \"11dc1e70-0522-4d3f-9b48-e43c5e60b88f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.333555212Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.34573113Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/2e9d3399-1c88-40d6-9437-68400e50f843/snapshots/11dc1e70-0522-4d3f-9b48-e43c5e60b88f","method":"GET","req_id":"7a3e342c-03b7-4825-84fe-60b6a45b8164","remote_addr":"[fd00:1122:3344:101::a]:60447","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":33673,"response_code":"200"}
{"msg":"path is \"/data/regions/2e9d3399-1c88-40d6-9437-68400e50f843\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.346500763Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"dataset is oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.355328823Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"snapshot is oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843@11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.36452404Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/2e9d3399-1c88-40d6-9437-68400e50f843\", \"11dc1e70-0522-4d3f-9b48-e43c5e60b88f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.364552721Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2e9d3399-1c88-40d6-9437-68400e50f843-11dc1e70-0522-4d3f-9b48-e43c5e60b88f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.374228146Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"applying SMF actions for region 2e9d3399-1c88-40d6-9437-68400e50f843 running snapshot 11dc1e70-0522-4d3f-9b48-e43c5e60b88f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.374757115Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.374790755Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/2e9d3399-1c88-40d6-9437-68400e50f843/snapshots/11dc1e70-0522-4d3f-9b48-e43c5e60b88f/run","method":"POST","req_id":"d69d92a2-c2a1-4b27-8948-d0ec3ef8d6c2","remote_addr":"[fd00:1122:3344:101::a]:60447","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28942,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380144597Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380232118Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.38031566Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.38032316Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.38036709Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380418961Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380516943Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380583964Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380635095Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.380687846Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.386984923Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2e9d3399-1c88-40d6-9437-68400e50f843-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.387464171Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2e9d3399-1c88-40d6-9437-68400e50f843-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.39382683Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2e9d3399-1c88-40d6-9437-68400e50f843-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406254362Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843/.zfs/snapshot/11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406445495Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406498636Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406539817Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406578727Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.406628888Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.409877393Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.414623865Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"region 2e9d3399-1c88-40d6-9437-68400e50f843 running snapshot 11dc1e70-0522-4d3f-9b48-e43c5e60b88f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.414643015Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.09425723Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57799"}
{"msg":"region f5416a9b-bfdb-4482-af5a-00dc0c7796a3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.094956342Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.095110764Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.095124155Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions","method":"POST","req_id":"0e91fc0b-4fa9-4c75-aad1-ec5bb37d7a6d","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":815,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.105139226Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.105161146Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f5416a9b-bfdb-4482-af5a-00dc0c7796a3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.170109783Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"f5416a9b-bfdb-4482-af5a-00dc0c7796a3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f5416a9b-bfdb-4482-af5a-00dc0c7796a3\" \"--data\" \"/data/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.170177604Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"f5416a9b-bfdb-4482-af5a-00dc0c7796a3","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.238918177Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"f5416a9b-bfdb-4482-af5a-00dc0c7796a3","component":"worker"}
{"msg":"region f5416a9b-bfdb-4482-af5a-00dc0c7796a3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.238971878Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.2390835Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239658469Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239739351Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239818682Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239826562Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239865943Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.239915474Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.24028754Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.240351861Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.240402372Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.240450803Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.250708668Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.251157596Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.257610266Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.264511983Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.267212209Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.267484514Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.267552985Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.267602176Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.267650417Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.27138031Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.276852174Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.443555577Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions","method":"POST","req_id":"a4a2d4b1-06f3-472e-b9c3-78b611578e52","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.700031988Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","method":"GET","req_id":"f04df52d-31c7-4625-ad4b-ec769f2a0998","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"path is \"/data/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.702713914Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"dataset is oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.737233222Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"region f5416a9b-bfdb-4482-af5a-00dc0c7796a3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.745530804Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.745711397Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.745774908Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","method":"DELETE","req_id":"12242014-a708-486d-87a6-19b234b2e543","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":43083,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f5416a9b-bfdb-4482-af5a-00dc0c7796a3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.746960978Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.749170236Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","method":"GET","req_id":"0ac60991-1675-4349-b2cf-47b9a605f982","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":35,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.75963542Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.759728862Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.759809995Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.759829682Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.759847739Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.759900493Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e9d3399-1c88-40d6-9437-68400e50f843","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.760316496Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.760385787Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.760447428Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.760509309Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.769630745Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.771738921Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crucible/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.780983648Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"region":"f5416a9b-bfdb-4482-af5a-00dc0c7796a3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.849511407Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","method":"GET","req_id":"f0a477a6-522b-4149-8d4e-6399ac2d9b5f","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":124,"response_code":"200"}
{"msg":"region f5416a9b-bfdb-4482-af5a-00dc0c7796a3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.870745919Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.975695369Z","hostname":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","pid":29311,"uri":"/crucible/0/regions/f5416a9b-bfdb-4482-af5a-00dc0c7796a3","method":"GET","req_id":"4693a51e-0d6c-468f-880c-ab2e6e9787d5","remote_addr":"[fd00:1122:3344:101::a]:57799","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20,"response_code":"200"}