[ Jul  5 23:25:58 Enabled. ]
[ Jul  5 23:25:58 Rereading configuration. ]
[ Jul  5 23:25:59 Rereading configuration. ]
[ Jul  5 23:26:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:26:01.111841419Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:26:01.117047906Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:26:01.117064936Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:26:01.131680964Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:26:01.215296846Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.944524516Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:60774"}
{"msg":"region 5c60a764-5861-44a4-8500-88ccfaf93eab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.95403232Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.95447466Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions","method":"POST","req_id":"86ca9ed4-7231-4e51-b141-cf3febd82fe1","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":4740,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.956581328Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.966151203Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:09.966186534Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5c60a764-5861-44a4-8500-88ccfaf93eab\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5c60a764-5861-44a4-8500-88ccfaf93eab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.018345917Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"region":"5c60a764-5861-44a4-8500-88ccfaf93eab","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5c60a764-5861-44a4-8500-88ccfaf93eab\" \"--data\" \"/data/regions/5c60a764-5861-44a4-8500-88ccfaf93eab\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.020796733Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"region":"5c60a764-5861-44a4-8500-88ccfaf93eab","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.158192434Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions","method":"POST","req_id":"e40b924b-f801-4e1f-a6ad-03106927bbfb","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.281854697Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"region":"5c60a764-5861-44a4-8500-88ccfaf93eab","component":"worker"}
{"msg":"region 5c60a764-5861-44a4-8500-88ccfaf93eab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.281914698Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.28201992Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.28243875Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.287111565Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.297228902Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.300530937Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.301232673Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.301374096Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.301428047Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.301467458Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.305942308Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.318816288Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:10.701616992Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions","method":"POST","req_id":"6027bed7-2ec7-409a-bcbf-e33d52959f64","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.365108602Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","method":"GET","req_id":"cdf204ba-494b-45a7-bf79-98f9e1dac541","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"path is \"/data/regions/5c60a764-5861-44a4-8500-88ccfaf93eab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.367962596Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"dataset is oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.378022713Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"region 5c60a764-5861-44a4-8500-88ccfaf93eab state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.387874504Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.388041228Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.38811047Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","method":"DELETE","req_id":"f477b650-783c-4c0e-8076-7830118e6f3b","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20765,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-5c60a764-5861-44a4-8500-88ccfaf93eab (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.389122973Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.391647579Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","method":"GET","req_id":"5dd60a2b-d6e3-45d0-b33d-e614f6c5e049","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.39301995Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9278e74e-8bb6-4f30-99ac-739a13623327/crucible/regions/5c60a764-5861-44a4-8500-88ccfaf93eab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.403043666Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"region":"5c60a764-5861-44a4-8500-88ccfaf93eab","component":"worker"}
{"msg":"region 5c60a764-5861-44a4-8500-88ccfaf93eab state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.454275459Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-05T23:27:11.533728887Z","hostname":"oxz_crucible_afd9028a-666a-46d0-984c-5e2768e06355","pid":21024,"uri":"/crucible/0/regions/5c60a764-5861-44a4-8500-88ccfaf93eab","method":"GET","req_id":"88a4abc0-500c-4d22-9c1d-c9afea0d8e4e","remote_addr":"[fd00:1122:3344:101::b]:60774","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}