[ Dec 19 05:53:17 Enabled. ]
[ Dec 19 05:53:17 Rereading configuration. ]
[ Dec 19 05:53:20 Rereading configuration. ]
[ Dec 19 05:53:24 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:53:25.039462915Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:53:25.042797842Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:53:25.042807292Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:53:25.058672763Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:53:25.142924318Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.299340762Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60192"}
{"msg":"region 31f45dfc-a64e-44dc-a756-9659660ac964 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.320898107Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.32534458Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.329780103Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions","method":"POST","req_id":"56742146-48f3-435b-9934-aa35635e72f0","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":10942,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible/regions/31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.335760885Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible/regions/31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.335787116Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"31f45dfc-a64e-44dc-a756-9659660ac964\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/31f45dfc-a64e-44dc-a756-9659660ac964\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.398668692Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"region":"31f45dfc-a64e-44dc-a756-9659660ac964","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"31f45dfc-a64e-44dc-a756-9659660ac964\" \"--data\" \"/data/regions/31f45dfc-a64e-44dc-a756-9659660ac964\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.400602614Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"region":"31f45dfc-a64e-44dc-a756-9659660ac964","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.483813195Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions","method":"POST","req_id":"e284c92d-6a15-4da3-914a-8d89cb3d80a5","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.509641097Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"region":"31f45dfc-a64e-44dc-a756-9659660ac964","component":"worker"}
{"msg":"region 31f45dfc-a64e-44dc-a756-9659660ac964 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.509673748Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.509780372Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.510154124Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.515241148Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.525226199Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.528081031Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.528787094Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.528856516Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.528902508Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.528942299Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.532598717Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.537187535Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:14.759347762Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions","method":"POST","req_id":"66cd1b81-ba6f-4a2d-a77d-8ea525a2aef7","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.576774236Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions/31f45dfc-a64e-44dc-a756-9659660ac964","method":"GET","req_id":"a441bcb5-bfee-4282-a16a-6e8cd5341ddc","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"path is \"/data/regions/31f45dfc-a64e-44dc-a756-9659660ac964\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.58094183Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"dataset is oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible/regions/31f45dfc-a64e-44dc-a756-9659660ac964","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.592015317Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"region 31f45dfc-a64e-44dc-a756-9659660ac964 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.60112033Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.601260155Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.601318177Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions/31f45dfc-a64e-44dc-a756-9659660ac964","method":"DELETE","req_id":"21703992-ecbd-475e-9339-3c09228b315f","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20404,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-31f45dfc-a64e-44dc-a756-9659660ac964 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.60234803Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.604592662Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions/31f45dfc-a64e-44dc-a756-9659660ac964","method":"GET","req_id":"8f226caf-5862-497e-b894-5f2fe68b3ce5","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.608925942Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0cdd8476-a0d8-44ca-a74b-0704c56ede98/crucible/regions/31f45dfc-a64e-44dc-a756-9659660ac964\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.620917868Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"region":"31f45dfc-a64e-44dc-a756-9659660ac964","component":"worker"}
{"msg":"region 31f45dfc-a64e-44dc-a756-9659660ac964 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.681443258Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T05:54:15.693023451Z","hostname":"oxz_crucible_e027a9c4-2388-40aa-8459-5f776f88554d","pid":24875,"uri":"/crucible/0/regions/31f45dfc-a64e-44dc-a756-9659660ac964","method":"GET","req_id":"2154d53d-86f5-4e28-ab8e-90a7b1741d2d","remote_addr":"[fd00:1122:3344:101::c]:60192","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":71,"response_code":"200"}