[ Jun 22 01:58:51 Enabled. ]
[ Jun 22 01:58:51 Rereading configuration. ]
[ Jun 22 01:58:52 Rereading configuration. ]
[ Jun 22 01:58:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.503200328Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.503564373Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.503574373Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.526303757Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.613936234Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.654549569Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60878"}
{"msg":"region 1560a552-93f7-4dfb-8d03-00f909dcee98 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.656543139Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.657521484Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions","method":"POST","req_id":"47f127d5-38fa-4600-92d3-849dbec85f86","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":1305,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.657545875Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.667206031Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.667238991Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1560a552-93f7-4dfb-8d03-00f909dcee98\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1560a552-93f7-4dfb-8d03-00f909dcee98\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.717299429Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"region":"1560a552-93f7-4dfb-8d03-00f909dcee98","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1560a552-93f7-4dfb-8d03-00f909dcee98\" \"--data\" \"/data/regions/1560a552-93f7-4dfb-8d03-00f909dcee98\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.71737376Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"region":"1560a552-93f7-4dfb-8d03-00f909dcee98","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.796898874Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions","method":"POST","req_id":"526dfa6b-4c0a-448a-87c7-d1cb091ebb98","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.836336462Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"region":"1560a552-93f7-4dfb-8d03-00f909dcee98","component":"worker"}
{"msg":"region 1560a552-93f7-4dfb-8d03-00f909dcee98 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.836384542Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.836502744Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.836968011Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.84217321Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.849987678Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.852906062Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.853644013Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.853735385Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.853779235Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.853817526Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.858926404Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.863927729Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.446492329Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions","method":"POST","req_id":"376e2209-0f42-498b-9a71-598e633b1467","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":802,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.995889437Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","method":"GET","req_id":"32f898b8-a64f-44dc-bf44-58710deba0e4","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"path is \"/data/regions/1560a552-93f7-4dfb-8d03-00f909dcee98\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.998646089Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"dataset is oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.008460418Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"region 1560a552-93f7-4dfb-8d03-00f909dcee98 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.018156304Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.01915063Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.019230091Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","method":"DELETE","req_id":"7f22c12e-65db-4f90-a3f5-f80d01876433","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20641,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-1560a552-93f7-4dfb-8d03-00f909dcee98 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.020225186Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.023076649Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","method":"GET","req_id":"de53ef10-9243-42b4-bb00-2d3f04e9a64a","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.026111005Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crucible/regions/1560a552-93f7-4dfb-8d03-00f909dcee98\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.036486422Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"region":"1560a552-93f7-4dfb-8d03-00f909dcee98","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.080761662Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","method":"GET","req_id":"a52afea1-9423-40d4-a40e-b7a6a715ca2b","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":879,"response_code":"200"}
{"msg":"region 1560a552-93f7-4dfb-8d03-00f909dcee98 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.08592651Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:15.24371411Z","hostname":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","pid":24407,"uri":"/crucible/0/regions/1560a552-93f7-4dfb-8d03-00f909dcee98","method":"GET","req_id":"100e2d64-0f3b-4972-9d94-b7e6eca5389c","remote_addr":"[fd00:1122:3344:101::c]:60878","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":46,"response_code":"200"}