[ May 31 00:14:26 Enabled. ]
[ May 31 00:14:26 Rereading configuration. ]
[ May 31 00:14:28 Rereading configuration. ]
[ May 31 00:14:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:36.491392736Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:36.494800012Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:36.494812372Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:36.51123978Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:36.628470181Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.66379027Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64095"}
{"msg":"region f246c40f-f7a2-40ca-bef5-d4fb94e992f0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.689550616Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.69404032Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions","method":"POST","req_id":"17328c9c-1e1a-4c25-8928-4c57a623e74b","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":11087,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.6940678Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.704251739Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.70429225Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f246c40f-f7a2-40ca-bef5-d4fb94e992f0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.786356367Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"f246c40f-f7a2-40ca-bef5-d4fb94e992f0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f246c40f-f7a2-40ca-bef5-d4fb94e992f0\" \"--data\" \"/data/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.788922002Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"f246c40f-f7a2-40ca-bef5-d4fb94e992f0","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:53.877609836Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions","method":"POST","req_id":"91f4c658-6da4-4929-bf7b-e0bc0528499b","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":899,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.101309813Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"f246c40f-f7a2-40ca-bef5-d4fb94e992f0","component":"worker"}
{"msg":"region f246c40f-f7a2-40ca-bef5-d4fb94e992f0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.101359365Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.101449797Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.101857647Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.109551513Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.123402385Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.127877199Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.128536906Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.130806893Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.130868015Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.130909166Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.139415402Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.144981834Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.154894916Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions","method":"POST","req_id":"ab027387-51c8-4424-9698-49b5a9282a47","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":764,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.766998168Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","method":"GET","req_id":"5cb0d2e5-f269-451e-b47f-92398194dc29","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"path is \"/data/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.76943279Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"dataset is oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.779139017Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"region f246c40f-f7a2-40ca-bef5-d4fb94e992f0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.787636743Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.787774926Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.787853919Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","method":"DELETE","req_id":"b6c97187-0632-4b8f-af59-3d8eda420cec","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18475,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f246c40f-f7a2-40ca-bef5-d4fb94e992f0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.788921696Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.790468705Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","method":"GET","req_id":"cdd3bc42-9898-43b2-bcda-da0f2d26036e","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.80521406Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.815592024Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"f246c40f-f7a2-40ca-bef5-d4fb94e992f0","component":"worker"}
{"msg":"region f246c40f-f7a2-40ca-bef5-d4fb94e992f0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.880231316Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:54.927409996Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions/f246c40f-f7a2-40ca-bef5-d4fb94e992f0","method":"GET","req_id":"c0efa002-0782-4499-a48f-13107afcf6b3","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region fe9f7219-01ea-4838-ad6f-2813ecea51b9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.643384453Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.643531197Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions","method":"POST","req_id":"ad843ef9-9fe1-4c0f-b7b6-d62a752e862c","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":217,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.643559668Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.653789428Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_34c18331-fe40-4cba-a36f-09dd53cc878a/crucible/regions/fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.653819568Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fe9f7219-01ea-4838-ad6f-2813ecea51b9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fe9f7219-01ea-4838-ad6f-2813ecea51b9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.721204832Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"fe9f7219-01ea-4838-ad6f-2813ecea51b9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fe9f7219-01ea-4838-ad6f-2813ecea51b9\" \"--data\" \"/data/regions/fe9f7219-01ea-4838-ad6f-2813ecea51b9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.723155731Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"fe9f7219-01ea-4838-ad6f-2813ecea51b9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.817768937Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"region":"fe9f7219-01ea-4838-ad6f-2813ecea51b9","component":"worker"}
{"msg":"region fe9f7219-01ea-4838-ad6f-2813ecea51b9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.817816908Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.81788722Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.819399888Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.827299189Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.836808681Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.841169312Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fe9f7219-01ea-4838-ad6f-2813ecea51b9","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.841376377Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.841427068Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.841466409Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.84150355Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.846446376Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.851603947Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-31T00:14:56.932961105Z","hostname":"oxz_crucible_670c9c2d-68d3-42be-bdea-f11233d449b1","pid":24023,"uri":"/crucible/0/regions","method":"POST","req_id":"4039715e-e570-40d0-85bf-2166461b28b0","remote_addr":"[fd00:1122:3344:101::a]:64095","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}