[ Jul 10 20:02:45 Enabled. ]
[ Jul 10 20:02:45 Rereading configuration. ]
[ Jul 10 20:02:46 Rereading configuration. ]
[ Jul 10 20:02:47 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.153127251Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.155909535Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.155924885Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.175179474Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.325598491Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.14447911Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60083"}
{"msg":"region 2e1f122c-584a-47c0-abd1-ab21c0d50084 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.170166563Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.177414683Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"9154c3dc-ac30-410d-b6be-6ee0044e44dd","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16717,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.179675902Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.193409075Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.193445376Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2e1f122c-584a-47c0-abd1-ab21c0d50084\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.250382097Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"region":"2e1f122c-584a-47c0-abd1-ab21c0d50084","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2e1f122c-584a-47c0-abd1-ab21c0d50084\" \"--data\" \"/data/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.252772817Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"region":"2e1f122c-584a-47c0-abd1-ab21c0d50084","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.382884066Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"eefaf222-ed80-495c-a85e-cb3eb50ff178","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.54272535Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"region":"2e1f122c-584a-47c0-abd1-ab21c0d50084","component":"worker"}
{"msg":"region 2e1f122c-584a-47c0-abd1-ab21c0d50084 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.542774721Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.542890702Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.543385516Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.550612536Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.567067372Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.569573753Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.570304669Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.575220549Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.57528072Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.57532133Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.582715622Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.589929091Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:25.8818952Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions","method":"POST","req_id":"88a33537-f708-4729-a9d0-6d9a77e8d157","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.382501658Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","method":"GET","req_id":"263ac84f-a809-479a-a1d3-9117aa7bc5b7","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":116,"response_code":"200"}
{"msg":"path is \"/data/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.386903985Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"dataset is oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.398899674Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"region 2e1f122c-584a-47c0-abd1-ab21c0d50084 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.408827466Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.409020808Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.409085909Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","method":"DELETE","req_id":"1918fe2f-ed52-4ad0-96e6-0fe88b7e847a","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23126,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-2e1f122c-584a-47c0-abd1-ab21c0d50084 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.410257758Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.412299785Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","method":"GET","req_id":"f6645b27-1b44-4ca6-b668-1a38c8050805","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.416031776Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5f19e68a-1267-4a24-b7a2-b30c47414e2a/crucible/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.425344013Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"region":"2e1f122c-584a-47c0-abd1-ab21c0d50084","component":"worker"}
{"msg":"region 2e1f122c-584a-47c0-abd1-ab21c0d50084 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.472177871Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:49.554908607Z","hostname":"oxz_crucible_5af84bcd-d843-4bd8-987a-dabf423750bc","pid":24987,"uri":"/crucible/0/regions/2e1f122c-584a-47c0-abd1-ab21c0d50084","method":"GET","req_id":"0d0afd13-3b6d-4d1e-9539-bc6eea4040f3","remote_addr":"[fd00:1122:3344:101::a]:60083","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":57,"response_code":"200"}