[ Jun 21 21:41:18 Enabled. ]
[ Jun 21 21:41:18 Rereading configuration. ]
[ Jun 21 21:41:19 Rereading configuration. ]
[ Jun 21 21:41:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:21.189797827Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:21.193223505Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:21.193252216Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:21.204208783Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:21.306526332Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.544207649Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44660"}
{"msg":"region b35455ad-9bf1-4d91-98a2-a3efb296805f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.575466919Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.580722854Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.580738904Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions","method":"POST","req_id":"fbe018d6-ab6e-4486-a782-285b0610b1b5","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":10006,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.594403785Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.594511627Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.766206493Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions","method":"POST","req_id":"055e6493-c3f8-4238-b91b-21c55d85ec93","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"creating region Region { id: RegionId(\"b35455ad-9bf1-4d91-98a2-a3efb296805f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.766683602Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"region":"b35455ad-9bf1-4d91-98a2-a3efb296805f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b35455ad-9bf1-4d91-98a2-a3efb296805f\" \"--data\" \"/data/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:45.769049649Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"region":"b35455ad-9bf1-4d91-98a2-a3efb296805f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.353252868Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"region":"b35455ad-9bf1-4d91-98a2-a3efb296805f","component":"worker"}
{"msg":"region b35455ad-9bf1-4d91-98a2-a3efb296805f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.35335055Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.353535254Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.353940362Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.362697905Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.375977509Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.387025858Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.396326163Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.398430434Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.398487975Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.398534416Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.410097926Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.419492522Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:46.424182475Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions","method":"POST","req_id":"c9a436ff-4242-4702-8074-62f4c0907129","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.91209779Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","method":"GET","req_id":"27a86779-7009-4d8e-8d3d-e4671c6c4627","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"path is \"/data/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.914940216Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"dataset is oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.924256461Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"region b35455ad-9bf1-4d91-98a2-a3efb296805f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.932130828Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.932346832Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-b35455ad-9bf1-4d91-98a2-a3efb296805f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.933660398Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.940529234Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","method":"DELETE","req_id":"a452a3f6-3a62-4290-9369-fe0e001e84ac","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26093,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.942906681Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","method":"GET","req_id":"74fe7d29-9060-471c-af72-203aac7da786","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.952682995Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crucible/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:47.961911168Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"region":"b35455ad-9bf1-4d91-98a2-a3efb296805f","component":"worker"}
{"msg":"region b35455ad-9bf1-4d91-98a2-a3efb296805f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:48.046239651Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-21T21:41:48.058619757Z","hostname":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","pid":22508,"uri":"/crucible/0/regions/b35455ad-9bf1-4d91-98a2-a3efb296805f","method":"GET","req_id":"cb74f8b2-7a16-4604-a7da-4ac813635d65","remote_addr":"[fd00:1122:3344:101::c]:44660","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":71,"response_code":"200"}