[ Jun 22 01:58:50 Enabled. ]
[ Jun 22 01:58:50 Rereading configuration. ]
[ Jun 22 01:58:52 Rereading configuration. ]
[ Jun 22 01:58:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.047165895Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.047587621Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.047598571Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.078706542Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:58:54.208520087Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.656699292Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54849"}
{"msg":"region f90269ec-4476-4d62-8a05-f1a6869dd58e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.658937885Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.65990466Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions","method":"POST","req_id":"2b2354c9-26bf-4cdc-9798-e28e341c3baf","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1294,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.659921971Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.66979228Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.66983243Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f90269ec-4476-4d62-8a05-f1a6869dd58e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.724131483Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"region":"f90269ec-4476-4d62-8a05-f1a6869dd58e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f90269ec-4476-4d62-8a05-f1a6869dd58e\" \"--data\" \"/data/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.724213304Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"region":"f90269ec-4476-4d62-8a05-f1a6869dd58e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.795880329Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions","method":"POST","req_id":"86178d53-bd23-498d-b93b-aa3bd207443c","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.846684098Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"region":"f90269ec-4476-4d62-8a05-f1a6869dd58e","component":"worker"}
{"msg":"region f90269ec-4476-4d62-8a05-f1a6869dd58e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.846741789Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.84683731Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.847327438Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.850647128Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.858576578Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.861801097Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.862486998Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.86263436Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.86268173Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.862721861Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.86728164Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:13.871591815Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.311590657Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions","method":"POST","req_id":"276684b2-2198-4b46-b32d-8ff2f86f7a66","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.837134584Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","method":"GET","req_id":"91f876e8-36ad-4327-91f2-40fe99995d36","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.840045988Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"dataset is oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.849706844Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"region f90269ec-4476-4d62-8a05-f1a6869dd58e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.858359355Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.858537788Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.858625809Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","method":"DELETE","req_id":"d028eef0-47a8-416e-877e-41e50ea3e116","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18640,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f90269ec-4476-4d62-8a05-f1a6869dd58e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.859784237Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.86267021Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","method":"GET","req_id":"e993b275-0beb-4d15-915e-03f4a42ead8b","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":129,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.864178973Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crucible/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.87384659Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"region":"f90269ec-4476-4d62-8a05-f1a6869dd58e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.917404189Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","method":"GET","req_id":"3be8a46c-3e96-44de-899b-aa9dc6dea309","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region f90269ec-4476-4d62-8a05-f1a6869dd58e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.943708017Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-22T01:59:14.982812499Z","hostname":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","pid":24184,"uri":"/crucible/0/regions/f90269ec-4476-4d62-8a05-f1a6869dd58e","method":"GET","req_id":"8505f872-2a86-4a40-8f02-0aa2bb722410","remote_addr":"[fd00:1122:3344:101::c]:54849","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":591,"response_code":"200"}