[ May  6 18:53:06 Enabled. ]
[ May  6 18:53:06 Rereading configuration. ]
[ May  6 18:53:10 Rereading configuration. ]
[ May  6 18:53:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.495768137Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.496576587Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.496588327Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.511574757Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.6162992Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.880082102Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47603"}
{"msg":"region 6c0353c8-55b9-49e9-8832-9258d08a2347 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.912950081Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.91539938Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"a2e4459c-8eb0-4649-9cf9-cc8be332f603","remote_addr":"[fd00:1122:3344:101::a]:47603","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":8659,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.917358487Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.926361043Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.926394444Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6c0353c8-55b9-49e9-8832-9258d08a2347\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.975505453Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"6c0353c8-55b9-49e9-8832-9258d08a2347","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c0353c8-55b9-49e9-8832-9258d08a2347\" \"--data\" \"/data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.977431379Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"6c0353c8-55b9-49e9-8832-9258d08a2347","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.118538916Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"6c0353c8-55b9-49e9-8832-9258d08a2347","component":"worker"}
{"msg":"region 6c0353c8-55b9-49e9-8832-9258d08a2347 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.118590377Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.118670349Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.119064819Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.123378082Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.126520827Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"894a7f18-835a-4313-b122-c9ced0cb6822","remote_addr":"[fd00:1122:3344:101::a]:47603","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.138145286Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.141727872Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.1424422Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.144375856Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.144428417Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.144467618Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.153402503Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.157126252Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.465425594Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33638"}
{"msg":"region cceb356c-6b85-434e-a9dd-f5a1d5bed655 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.495297821Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.499127853Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"1d1771a5-cac9-4616-9848-6f1ada027126","remote_addr":"[fd00:1122:3344:101::a]:33638","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3865,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.499149633Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.510386883Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.510407934Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cceb356c-6b85-434e-a9dd-f5a1d5bed655\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cceb356c-6b85-434e-a9dd-f5a1d5bed655\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.569331978Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"cceb356c-6b85-434e-a9dd-f5a1d5bed655","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cceb356c-6b85-434e-a9dd-f5a1d5bed655\" \"--data\" \"/data/regions/cceb356c-6b85-434e-a9dd-f5a1d5bed655\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.571241134Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"cceb356c-6b85-434e-a9dd-f5a1d5bed655","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.633912168Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"cceb356c-6b85-434e-a9dd-f5a1d5bed655","component":"worker"}
{"msg":"region cceb356c-6b85-434e-a9dd-f5a1d5bed655 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.633954339Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.634046651Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.644930022Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645042315Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645147858Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645159528Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645212489Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645275581Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645375383Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645444535Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645494166Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.645543127Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.651008058Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.651416578Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.655028475Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.664555003Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.667479404Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"9cb3a0eb-c900-4686-8003-ebf1b64695a2","remote_addr":"[fd00:1122:3344:101::a]:33638","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.667692949Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cceb356c-6b85-434e-a9dd-f5a1d5bed655","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.667972285Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668036357Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668083728Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668124199Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.671838358Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.676175072Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.274492484Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33037"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.274546705Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","method":"GET","req_id":"098fdfc1-eaf9-47b9-87e7-cd68ff9a5aae","remote_addr":"[fd00:1122:3344:101::a]:33037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.275278093Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"dataset is oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.284886564Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"snapshot is oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.292943507Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.292967467Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.307951237Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions/6c0353c8-55b9-49e9-8832-9258d08a2347/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","method":"GET","req_id":"afeea2d7-4304-4866-9a34-186db7b977d6","remote_addr":"[fd00:1122:3344:101::a]:33037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":33248,"response_code":"200"}
{"msg":"path is \"/data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.308116441Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"dataset is oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.316262277Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"snapshot is oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.324388442Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/6c0353c8-55b9-49e9-8832-9258d08a2347\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.324421282Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6c0353c8-55b9-49e9-8832-9258d08a2347-2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.333104261Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"applying SMF actions for region 6c0353c8-55b9-49e9-8832-9258d08a2347 running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.333288825Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.333310786Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions/6c0353c8-55b9-49e9-8832-9258d08a2347/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab/run","method":"POST","req_id":"0900a381-aade-474c-9158-66578433dfde","remote_addr":"[fd00:1122:3344:101::a]:33037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25218,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.333867789Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.333953991Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334034623Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334043563Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334082414Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334128975Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334466603Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334528095Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334576256Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.334622267Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.339172947Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-6c0353c8-55b9-49e9-8832-9258d08a2347-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.340548029Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6c0353c8-55b9-49e9-8832-9258d08a2347-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.345405036Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6c0353c8-55b9-49e9-8832-9258d08a2347-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357340172Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347/.zfs/snapshot/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357625259Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357685711Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357726132Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357767523Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.357822634Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.361132223Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.365272423Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"region 6c0353c8-55b9-49e9-8832-9258d08a2347 running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.365287713Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.004595204Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60792"}
{"msg":"region ca85dadb-64ac-4054-b517-c65310eaeb26 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034579324Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034722867Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"3e333ce1-9f0e-4613-8f37-791974b82594","remote_addr":"[fd00:1122:3344:101::a]:60792","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":795,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034746538Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.044241366Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a054c327-ba96-4046-8c5e-0adba1e47971/crucible/regions/ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.044268386Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ca85dadb-64ac-4054-b517-c65310eaeb26\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ca85dadb-64ac-4054-b517-c65310eaeb26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.106589142Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"ca85dadb-64ac-4054-b517-c65310eaeb26","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ca85dadb-64ac-4054-b517-c65310eaeb26\" \"--data\" \"/data/regions/ca85dadb-64ac-4054-b517-c65310eaeb26\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.108466897Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"ca85dadb-64ac-4054-b517-c65310eaeb26","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.170918846Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"region":"ca85dadb-64ac-4054-b517-c65310eaeb26","component":"worker"}
{"msg":"region ca85dadb-64ac-4054-b517-c65310eaeb26 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.170966987Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.17107923Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171737726Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171827198Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171930301Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171937511Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171982052Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.172040263Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0353c8-55b9-49e9-8832-9258d08a2347","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.172391482Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.172460613Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.172511895Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.172561306Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.17649576Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.176955311Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.180792003Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186975702Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.19192383Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ca85dadb-64ac-4054-b517-c65310eaeb26","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.192183867Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.192360251Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.192382181Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.192388962Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.196967622Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.207766201Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.389420691Z","hostname":"oxz_crucible_bb6f10d0-a184-4735-a3af-ca7d47b5ab96","pid":22869,"uri":"/crucible/0/regions","method":"POST","req_id":"96efdb36-2164-487c-a0d6-5bc63a559386","remote_addr":"[fd00:1122:3344:101::a]:60792","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}