[ Aug 22 17:49:12 Enabled. ]
[ Aug 22 17:49:12 Rereading configuration. ]
[ Aug 22 17:49:16 Rereading configuration. ]
[ Aug 22 17:49:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.555157744Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.557436606Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.557446826Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.569610884Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:49:22.7150133Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.451727107Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52013"}
{"msg":"region b4cd37c3-c416-4d1f-9b50-e94c5d9e911c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.47546407Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.479943763Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.484269705Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions","method":"POST","req_id":"cd221425-1ba4-4ab4-869f-970b11c6af5e","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16236,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.492396567Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.492421667Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.553288752Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"region":"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\" \"--data\" \"/data/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.555943401Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"region":"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.764443693Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"region":"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","component":"worker"}
{"msg":"region b4cd37c3-c416-4d1f-9b50-e94c5d9e911c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.764501244Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.764575734Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.764984716Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.769908101Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.789455452Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.792449678Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.797711828Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.800240738Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.800299998Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.800340669Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.803746966Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.807889978Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:02.827825275Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions","method":"POST","req_id":"633cdbb7-ea9d-464f-ac2c-b3a425ec4554","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.612710817Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","method":"GET","req_id":"e74b8f68-9e4d-468e-9042-c71d14e1411a","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":885,"response_code":"200"}
{"msg":"path is \"/data/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.616142284Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"dataset is oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.625509093Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"region b4cd37c3-c416-4d1f-9b50-e94c5d9e911c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.633532334Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.633668795Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.633742255Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","method":"DELETE","req_id":"227fca51-ab1f-48f4-838e-d25fab40a0e1","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":17626,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b4cd37c3-c416-4d1f-9b50-e94c5d9e911c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.634854081Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.636885272Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","method":"GET","req_id":"e00287fb-234e-41dc-8291-73c25fb6b0ad","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.639117583Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_38de2928-c5e7-4728-8a94-bef08f284b20/crucible/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.647190845Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"region":"b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","component":"worker"}
{"msg":"region b4cd37c3-c416-4d1f-9b50-e94c5d9e911c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.696280299Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-22T17:50:03.779455378Z","hostname":"oxz_crucible_7d40be40-71b4-46d9-963c-99c005bf304e","pid":22241,"uri":"/crucible/0/regions/b4cd37c3-c416-4d1f-9b50-e94c5d9e911c","method":"GET","req_id":"88db4102-d032-4726-8eb3-47af52a50e82","remote_addr":"[fd00:1122:3344:101::a]:52013","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":65,"response_code":"200"}