[ May 29 21:15:56 Enabled. ]
[ May 29 21:15:56 Rereading configuration. ]
[ May 29 21:15:58 Rereading configuration. ]
[ May 29 21:16:02 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:02.750738777Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:02.753404737Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:02.753423388Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:02.767500443Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:02.888840844Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.201716338Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:44895"}
{"msg":"region e4d9615e-d943-4829-8db3-5860291efc39 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.232014711Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.239231185Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.239271067Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"6185843f-47f5-4ac9-a64c-0acd28b41d1c","remote_addr":"[fd00:1122:3344:101::b]:44895","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15708,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.249985819Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.2500167Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e4d9615e-d943-4829-8db3-5860291efc39\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e4d9615e-d943-4829-8db3-5860291efc39\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.300764265Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"e4d9615e-d943-4829-8db3-5860291efc39","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e4d9615e-d943-4829-8db3-5860291efc39\" \"--data\" \"/data/regions/e4d9615e-d943-4829-8db3-5860291efc39\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.303067743Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"e4d9615e-d943-4829-8db3-5860291efc39","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.430123396Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"ada64398-2c15-424f-9589-d174bb073181","remote_addr":"[fd00:1122:3344:101::b]:44895","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.613251185Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"e4d9615e-d943-4829-8db3-5860291efc39","component":"worker"}
{"msg":"region e4d9615e-d943-4829-8db3-5860291efc39 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.613296036Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.61340583Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.613808154Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.620823861Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.634782122Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.639900706Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.648396083Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.650426521Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.650501754Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.650552895Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.654580122Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.658281937Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:03.968418628Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"47fc75d0-5afe-402a-b271-03d8dd459411","remote_addr":"[fd00:1122:3344:101::b]:44895","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":103,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.739692433Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:54891"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.739818747Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/e4d9615e-d943-4829-8db3-5860291efc39","method":"GET","req_id":"3baa6378-5ddd-4152-b36b-e95238989e41","remote_addr":"[fd00:1122:3344:101::b]:54891","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"path is \"/data/regions/e4d9615e-d943-4829-8db3-5860291efc39\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.739990073Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"dataset is oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/e4d9615e-d943-4829-8db3-5860291efc39","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.748805891Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"region e4d9615e-d943-4829-8db3-5860291efc39 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.757367181Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.757583778Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-e4d9615e-d943-4829-8db3-5860291efc39 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.758766248Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.759515103Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/e4d9615e-d943-4829-8db3-5860291efc39","method":"DELETE","req_id":"8b362a9d-0c23-4486-9583-3545bedb9270","remote_addr":"[fd00:1122:3344:101::b]:54891","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19564,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.759655608Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/e4d9615e-d943-4829-8db3-5860291efc39","method":"GET","req_id":"4d1e8c83-6094-4ebc-b5d7-6a1d7b740f81","remote_addr":"[fd00:1122:3344:101::b]:54891","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.76239403Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/e4d9615e-d943-4829-8db3-5860291efc39\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.771889542Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"e4d9615e-d943-4829-8db3-5860291efc39","component":"worker"}
{"msg":"region e4d9615e-d943-4829-8db3-5860291efc39 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.820644729Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:04.864050846Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/e4d9615e-d943-4829-8db3-5860291efc39","method":"GET","req_id":"1e8a0f0d-16f6-45f7-8210-b74acd66bdf0","remote_addr":"[fd00:1122:3344:101::b]:54891","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.374449738Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46297"}
{"msg":"region f85f08d7-9645-4055-b5ac-5d084b21550e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.374543882Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.374744588Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.374752998Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"75fc675c-b464-42e6-bde0-c68e997a66be","remote_addr":"[fd00:1122:3344:101::a]:46297","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":245,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.382685717Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.382711627Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f85f08d7-9645-4055-b5ac-5d084b21550e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.429158277Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"f85f08d7-9645-4055-b5ac-5d084b21550e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f85f08d7-9645-4055-b5ac-5d084b21550e\" \"--data\" \"/data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.429205359Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"f85f08d7-9645-4055-b5ac-5d084b21550e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.488747071Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"f85f08d7-9645-4055-b5ac-5d084b21550e","component":"worker"}
{"msg":"region f85f08d7-9645-4055-b5ac-5d084b21550e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.488797843Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.488885156Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.490459919Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.493554804Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.500592411Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.503477469Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.503719027Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.503769089Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.50380903Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.503847852Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.506908565Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.51090757Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.624771968Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"4eea007c-7b80-49bc-a6bb-a7a63b7a5cd1","remote_addr":"[fd00:1122:3344:101::a]:46297","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.542192882Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45688"}
{"msg":"region 1ffbca5b-daaa-4d5e-bd69-786e137cee35 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.542440651Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.542901466Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"13b6947f-e835-479f-b0d4-c48d91951353","remote_addr":"[fd00:1122:3344:101::a]:45688","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":687,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.542930917Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.551966813Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.551988243Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1ffbca5b-daaa-4d5e-bd69-786e137cee35\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1ffbca5b-daaa-4d5e-bd69-786e137cee35\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.603427632Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"1ffbca5b-daaa-4d5e-bd69-786e137cee35","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1ffbca5b-daaa-4d5e-bd69-786e137cee35\" \"--data\" \"/data/regions/1ffbca5b-daaa-4d5e-bd69-786e137cee35\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.603475663Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"1ffbca5b-daaa-4d5e-bd69-786e137cee35","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.698976111Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"90083b01-bea1-49f2-b4c8-2044b1cee5a1","remote_addr":"[fd00:1122:3344:101::a]:45688","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.715578272Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"region":"1ffbca5b-daaa-4d5e-bd69-786e137cee35","component":"worker"}
{"msg":"region 1ffbca5b-daaa-4d5e-bd69-786e137cee35 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.715594653Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.715741348Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.716755472Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.724450742Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.736758378Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.740208824Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1ffbca5b-daaa-4d5e-bd69-786e137cee35","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.740453063Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.740498984Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.740539786Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.740578517Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.744033704Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:21.749287751Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.243202224Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions","method":"POST","req_id":"723e9f63-d5b7-4a45-9d54-57ae962721b6","remote_addr":"[fd00:1122:3344:101::a]:45688","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":756,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.833290776Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39080"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.833349507Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","method":"GET","req_id":"6827eaa5-dc1a-4896-b42d-c4d4359cf568","remote_addr":"[fd00:1122:3344:101::a]:39080","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.836156502Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"dataset is oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.844531845Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"snapshot is oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e@dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.853650584Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\", \"dc5d364c-5dc8-4513-a3ee-bba5cdbfec48\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.853685525Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.865840076Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/f85f08d7-9645-4055-b5ac-5d084b21550e/snapshots/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","method":"GET","req_id":"8f58c2c3-16e6-4a04-96db-d8590b2f4d77","remote_addr":"[fd00:1122:3344:101::a]:39080","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":32344,"response_code":"200"}
{"msg":"path is \"/data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.866634762Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"dataset is oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.875030826Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"snapshot is oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e@dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.883075428Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_593581c4-ec6a-488f-becc-2e312ddcd711/crucible/regions/f85f08d7-9645-4055-b5ac-5d084b21550e\", \"dc5d364c-5dc8-4513-a3ee-bba5cdbfec48\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.883108109Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f85f08d7-9645-4055-b5ac-5d084b21550e-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.891092359Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}
{"msg":"applying SMF actions for region f85f08d7-9645-4055-b5ac-5d084b21550e running snapshot dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.891272875Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.891328597Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"uri":"/crucible/0/regions/f85f08d7-9645-4055-b5ac-5d084b21550e/snapshots/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48/run","method":"POST","req_id":"57953847-5595-45f0-b7f1-c241ddc23d02","remote_addr":"[fd00:1122:3344:101::a]:39080","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25363,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-f85f08d7-9645-4055-b5ac-5d084b21550e-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.893951266Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f85f08d7-9645-4055-b5ac-5d084b21550e-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.89880091Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f85f08d7-9645-4055-b5ac-5d084b21550e-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910115312Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f85f08d7-9645-4055-b5ac-5d084b21550e/.zfs/snapshot/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910380621Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910435333Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910480404Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910528446Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.910587948Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.914845072Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.918545097Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"worker"}
{"msg":"region f85f08d7-9645-4055-b5ac-5d084b21550e running snapshot dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.918555807Z","hostname":"oxz_crucible_e3df8282-294b-4299-b727-a897e5db6212","pid":26355,"component":"datafile"}