[ Aug 29 15:18:39 Enabled. ]
[ Aug 29 15:18:39 Rereading configuration. ]
[ Aug 29 15:18:40 Rereading configuration. ]
[ Aug 29 15:18:42 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:18:42.523426022Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:18:42.524568235Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:18:42.524580015Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:18:42.545839041Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:18:42.655143312Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.623504779Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36168"}
{"msg":"region edea7955-ea23-4fca-890c-e35d7af3c3d8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.635495594Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.637887411Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions","method":"POST","req_id":"eb90c9e0-93ab-4625-b93f-c52e3b6f59bb","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":6650,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.638492933Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.648782164Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.648858606Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"edea7955-ea23-4fca-890c-e35d7af3c3d8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.727396651Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"region":"edea7955-ea23-4fca-890c-e35d7af3c3d8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"edea7955-ea23-4fca-890c-e35d7af3c3d8\" \"--data\" \"/data/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.729349832Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"region":"edea7955-ea23-4fca-890c-e35d7af3c3d8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.779916702Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions","method":"POST","req_id":"3cc44733-63bd-4ef1-bb3f-13d1878f7d5a","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.85232357Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"region":"edea7955-ea23-4fca-890c-e35d7af3c3d8","component":"worker"}
{"msg":"region edea7955-ea23-4fca-890c-e35d7af3c3d8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.852394981Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.852662427Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.853152216Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.856701686Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.870443184Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.874242949Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.879584823Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.881587323Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.881640354Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.881685785Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.886837365Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:16.890378115Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.161966833Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions","method":"POST","req_id":"402da11a-d3a6-4597-ab0b-ede5505ce886","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.860586442Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","method":"GET","req_id":"91eee719-e1df-4487-ac22-3e58f3ddc597","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.864125031Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"dataset is oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.87380583Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"region edea7955-ea23-4fca-890c-e35d7af3c3d8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.882255446Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.88246874Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.882531821Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","method":"DELETE","req_id":"bd665518-f98a-4b24-ba1b-46340a083efb","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19067,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-edea7955-ea23-4fca-890c-e35d7af3c3d8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.883658223Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.885793725Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","method":"GET","req_id":"a0c7ee36-1acb-4911-847e-1df8606e7528","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.888108561Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.89830574Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"region":"edea7955-ea23-4fca-890c-e35d7af3c3d8","component":"worker"}
{"msg":"region edea7955-ea23-4fca-890c-e35d7af3c3d8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.96261648Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-29T15:19:17.993322041Z","hostname":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","pid":26282,"uri":"/crucible/0/regions/edea7955-ea23-4fca-890c-e35d7af3c3d8","method":"GET","req_id":"65b73b08-bd7c-4602-9194-a7a7ffb4af57","remote_addr":"[fd00:1122:3344:101::a]:36168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}