linux.adoc

:toc:
:toclevels: 10
:toc-title: Contents
:toc-class: floating-toc

The initial implementation handles the debug (-D) option, with more
common options to be migrated in future patches.

= Costa's Linux kernel research scratchpad

keywords:

	CONFIG_ACPI_HOTPLUG_CPU

	
	scale_freq_tick
		"Scheduler frequency invariance went wobbly, disabling!"

	task_work_run


	notify_cpu_starting
		rcutree_report_cpu_starting

	vfs_rmdir
		kernfs_iop_rmdir
			cgroup_rmdir
				cgroup_destroy_locked
					kill_css
	
	cgroup_base_files
		cgroup_subtree_control_write
			cgroup_finalize_control
				cgroup_apply_control_disable
					kill_css

	kill_css
		percpu_ref_kill_and_confirm
		&css_killed_ref_fn
			offline_css
				cpuset_css_offline
					update_prstate ->
			css_put

       ktimers/7-94      [007] dN.11 110543.408319: ipi_send_cpu: cpu=7 callsite=irq_work_queue_on+0xe1/0xf0 callback=rcu_preempt_deferred_qs_handler+0x0/0x20	

	include/linux/context_tracking_irq.h

	__trace_stack
		ct_irq_enter_irqson
			ct_irq_enter
				ct_nmi_enter
					rcu_watching Startirq
		__ftrace_trace_stack
		ct_irq_exit_irqson
			ct_irq_exit
				ct_nmi_exit
					rcu_watching Endirq

	rcu_read_unlock_special
		@rcu_preempt_deferred_qs_handler
		init_irq_work
		irq_work_queue_on

	cpuset_write_resmask housekeeping_exlude_isolcpus cpu_up

	vim format paragpaph: gqip

== Intro

=== Table TOC

[cols="6*",options="header,footer,autowidth"]
|===
| <<human-interface>>
| <<system>>
| <<multitasking>>
| <<memory>>
| <<storage>>
| <<networking>>

|
<<debug>> +
<<video>>
|
<<smp>> +
<<cpumask>> +
<<cpuhp>> +
<<hw-interfaces,HW>>
|
<<real-time-preemption,RT>>
|
<<processes>> +
<<interrupts>>
|
<<storage-busses>> +
<<nvme>>
|
<<sockets>> +
<<ip>> +
<<eth>>

|
|
|
|
|
|

|
|
|
|
|
|
|===

=== TODO

	kernel:__common_interrupt: 32.57 No irq handler for vector

	sysctl kernel.sched_schedstats=1
	rtla hwnoise -H 2 -c 4,132 -d 1h --quiet -s 200 -t osnoise_trace.txt -p 1000000 -r1000000 -e sched -e timer -e irq_vectors -e irq -e nmi -e ipi -e power:cpu_frequency -e msr

	cryptomgr_init
		...
			cryptomgr_schedule_test
				cryptomgr_test
					alg_test
						alg_test_cipher
							test_cipher
								crypto_cipher_setkey
						alg_find_test

	timer_cancel

	tcp_timewait_state_process
		TCP_TIMEWAIT_LEN
		inet_twsk_reschedule
			__inet_twsk_schedule
				mod_timer
	inet_twsk_alloc
		timer_setup ->
		TIMER_PINNED
		tw_timer_handler
			inet_twsk_kill

	tcp_keepalive_timer

	debug_deactivate

	isolcpus
	non_isolcpus
	sched_load_balance
	/sys/kernel/debug/sched/domains
		update_sched_domain_debugfs
		SMT - Simultaneous multithreading
		SMT is the hyperthreading domain, so it's going to be just that CPU and its thread siblings
		MC is multi-core, should be all CPUs that share the same last-level cache

	newidle_balance

	normal_prio
		__normal_prio
	sched_core_enqueue
		rb_sched_core_less
			__sched_core_less
				prio_less
					__task_prio
						MAX_RT_PRIO
					dl_time_before
					cfs_prio_less
	rq_attach_root
		raw_spin_rq_lock_irqsave
			_raw_spin_rq_lock_irqsave
				raw_spin_rq_lock
					raw_spin_rq_lock_nested
						raw_spin_lock_nested
							_raw_spin_lock_nested
								do_raw_spin_trylock
									arch_spin_trylock
										queued_spin_trylock
								spin_acquire ->
	raw_spin_rq_unlock


	sysctl_timer_migration
		timers_migration_enabled

	timer_migration_handler
		timers_migration_enabled

	raw_smp_processor_id
		this_cpu_read(pcpu_hot.cpu_number)
		#define this_cpu_read(pcp)		__pcpu_size_call_return(this_cpu_read_, pcp)
	smp_processor_id
		__smp_processor_id
			__this_cpu_read(pcpu_hot.cpu_number)

	delayacct
	task_delay_info
	delayacct_init
	delayacct_end

	CONFIG_TICK_ONESHOT
		tick_irq_enter
		hotplug_cpu__broadcast_tick_pull
		tick_setup_hrtimer_broadcast
		tick-oneshot.o
			tick_program_event ...
		tick-sched.o
			tick_get_tick_sched
				tick_cpu_sched
			referrers tick_dep_set_.: tick_dep_set_cpu
				__perf_event_account_interrupt
					tick_dep_set_cpu
						tick_nohz_dep_set_cpu
				rcu_torture_reader
					tick_dep_set_task
				rcu_torture_fwd_prog_cr
					tick_dep_set_task
				__rcu_irq_enter_check_tick
					tick_dep_set_cpu
				rcu_do_batch
					tick_dep_set_task
				synchronize_rcu_expedited_wait
					tick_dep_set_cpu
				arm_timer
					tick_dep_set_task
					tick_dep_set_signal ->
				set_process_cpu_timer
					tick_dep_set_signal ->

	CONFIG_NO_HZ_COMMON
		TICK_ONESHOT
	CONFIG_NO_HZ_IDLE Idle dynticks system (tickless idle)
		(former NO_HZ "Old Idle dynticks config")
	CONFIG_NO_HZ_FULL Full dynticks system (tickless)
		CPU_ISOLATION
		CONFIG_IRQ_WORK
			irq_work_run
			irq_work_needs_cpu
			irq_work_tick_soft

	CONFIG_HZ_PERIODIC

	CONFIG_PREEMPT_RT PREEMPT_RT

		include/linux/interrupt.h
			force_irqthreads

		include/linux/rwlock_rt.h
			rt_read_lock
			...
		include/linux/rwlock_types.h
			rwlock_t

		kernel/locking/ww_rt_mutex.c


		include/linux/spinlock_rt.h
			spin_lock
			spin_lock_bh
				rt_spin_lock ->
			spin_lock_irq
				rt_spin_lock
		kernel/locking/spinlock_rt.c

		include/linux/rwbase_rt.h
			rwbase_rt
		kernel/locking/rwbase_rt.c


		rtmutex

		include/linux/rtmutex.h
			rt_mutex_init
			rt_mutex_lock
			rt_mutex
				rt_mutex_base
					raw_spinlock_t
		kernel/locking/rtmutex_common.h
		kernel/locking/rtmutex_api.c
			rt_mutex_base_init

			rt_mutex_lock
				__rt_mutex_lock_common
					mutex_acquire_nest
					__rt_mutex_lock
		kernel/locking/rtmutex.c
			__rt_mutex_lock
				rt_mutex_try_acquire
					rt_mutex_cmpxchg_acquire
						owner
                                                try_cmpxchg_acquire
                                                        raw_try_cmpxchg_acquire
                                                                cmpxchgl
				rt_mutex_slowlock
					__rt_mutex_slowlock_locked
						__rt_mutex_slowlock
							try_to_take_rt_mutex
								rt_mutex_owner
								rt_mutex_set_owner
                                                                        xchg_acquire
                                                                                xchg
			__rt_mutex_unlock
				rt_mutex_slowunlock
			...

		include/linux/spinlock_types.h
			spinlock_t
				rt_mutex_base
					raw_spinlock_t

		include/linux/spinlock_rt.h
		spin_lock
			rt_spin_lock
				spin_acquire ->
				__rt_spin_lock
					rtlock_lock
						rt_mutex_cmpxchg_acquire
						rtlock_slowlock
							rtlock_slowlock_locked
								try_to_take_rt_mutex ->
								schedule_rtlock
									schedule_loop(SM_RTLOCK_WAIT);
								raw_spin_lock_irq
									_raw_spin_lock_irq
		spin_unlock
			rt_spin_unlock
				spin_release
				migrate_enable
				rcu_read_unlock ->
				rt_mutex_cmpxchg_release
					try_cmpxchg_release
						raw_try_cmpxchg_release
                                        rt_mutex_has_waiters
                                        unlock_rt_mutex_safe
						clear_rt_mutex_waiters
						owner
                                                rt_mutex_cmpxchg_release
	                                mark_wakeup_next_waiter
						rt_mutex_top_waiter
						lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
						rt_mutex_wake_q_add
							rt_mutex_wake_q_add_task
                                        rt_mutex_wake_up_q
                                                wake_up_q


	rtla osnoise hist -d 5m -c 8 -Pf:$p
		include/linux/sched/mm.h
			mmdrop_sched
		&realtime_attr.attr
		./include/linux/local_lock_internal.h
			local_lock_t

		include/linux/mutex.h
			mutex_lock_nested
				used in acpi_scan_is_offline, dm_suspend, dm_resume ...
				__mutex_lock
					__mutex_lock_common ->
			struct mutex {
				struct rt_mutex_base	rtmutex;
			}

		./kernel/softirq.c
			softirq_ctrl ...
		./include/linux/sched/task.h
			put_task_struct
		./include/linux/sched.h
			schedule_rtlock
		./include/linux/preempt.h
			in_atomic
				!!preempt_count
			in_task
				(!((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | in_serving_softirq()))

			preempt_disable_nested
				preempt_disable
			preempt_enable_nested
				preempt_enable
			preempt_model_rt
		irq_exit_rcu
		__irq_exit_rcu
			wake_timersd
		if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
			cpu_relax();

	static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT);

	sched_init_debug
		debugfs_create_dir
		debugfs_create_file
		debugfs_create_u64

	PATCH: softirq: Wake ktimers thread also in softirq.
		If the hrtimer is raised
			while a softirq is processed
			then it does not wake the corresponding ktimers thread.

		This is due to the optimisation
			in the irq-exit path
			which is also used
			to wake the ktimers thread.
		__irq_exit_rcu [irq_exit]
			if (!in_interrupt() && local_softirq_pending())
				invoke_softirq();
			if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_pending_timers() &&
			    !(in_nmi() | in_hardirq()))
				wake_timersd();

	For the other softirqs, this is okay because the additional softirq bits will be handled by
	the currently running softirq handler.

	The timer related softirq bits are added to a different variable and rely on
	the ktimers thread.

	As a consuequence the wake up of ktimersd is delayed until the next timer tick.

	Always wake the ktimers thread if a timer related softirq is pending.

	PELT - Per-Entity Load Tracking

	echo timerlat > current_tracer


	brew mock-config --tag rhel-8.6.0-z-build --arch x86_64 -o ~/main/rhel-8.6.0.cfg
		-> config_opts['basedir'] = '/home/juril/mock'

	make BUILDID=".myid" rh-srpm

	mock -r /tmp/rhel-8.6.0.cfg /home/juril/rhel-8/redhat/rpm/SRPMS/kernel-rt-4.18.0-372.57.1.rt7.215.el8_6.myid.src.rpm

	ls ~/main/mock/rhel-8.6.0-z-build-repo_*/root/builddir/build/RPMS/

== System

=== SMP

	include/linux/smp.h kernel/smp.c arch/x86/kernel/smp.c

	include/linux/smp.h
		smp_send_stop
			smp_ops.stop_other_cpus
				native_stop_other_cpus
					apic_send_IPI_allbutself
		crash_smp_send_stop
			smp_ops.crash_stop_other_cpus
				kdump_nmi_shootdown_cpus
					nmi_shootdown_cpus
						apic_send_IPI_allbutself
	arch/x86/include/asm/smp.h
		stop_other_cpus and wait
			smp_ops.stop_other_cpus

	sysvec_call_function
		trace_call_function_entry(CALL_FUNCTION_VECTOR);
			"call_function_entry: vector=252"

	cpu_dev_init
		cpu_root_attrs

	kernel/smp.c
		smp_call_function
			<
				sysrq_showregs_othercpus
				kernel/scftorture.c
					scftorture_invoke_one
					scf_torture_cleanup
				kick_all_cpus_sync
				kernel/trace/ftrace.c	
					update_ftrace_function
					ftrace_modify_all_code
				rb_hammer_test
				tlb_remove_table_sync_one
				iucv_cleanup_queue
				arm
					bL_switcher_trace_trigger
					broadcast_tlb_a15_erratum
					__flush_ptrace_access
			cpu_online_mask
			smp_call_function_many
				can wait
				< 
					raise_mce
					__rwmsr_on_cpus
					arm broadcast_tlb_mm_a15_erratum
					drv_write
					scftorture_invoke_one
					membarrier_global_expedited
					membarrier_private_expedited
					clock_was_set
					kvm_kick_many_cpus
				smp_call_function_many_cond
					_RET_IP_
					trace_csd_queue_cpu
					send_call_function_single_ipi
						call_function_single_prep_ipi
						arch_send_call_function_single_ipi ->
					send_call_function_ipi_mask
						arch_send_call_function_ipi_mask
							kvm_smp_send_call_func_ipi
							native_send_call_func_ipi
								__apic_send_IPI_all
								__apic_send_IPI_mask ->
					csd_do_func
						trace_csd_function_entry
						trace_csd_function_exit
						include/trace/events/csd.h
					csd_lock_wait
						__csd_lock_wait
							csd_lock_wait_toolong
						smp_cond_load_acquire

		smp_call_function_single_async
			<
				crash_smp_send_stop
				raise_backtrace
				tick_broadcast
				zpci_handle_fallback_irq
				cpuid_read
				rdmsr_safe_on_cpu
				blk_mq_complete_send_ipi
				ingenic_tcu_cevt_cb
				cpuidle_coupled_poke
				liquidio_napi_drv_callback
				libeth_xsk_wakeup
				hrtick_start
				__unthrottle_cfs_rq_async
				kick_ilb
				__hrtimer_start_range_ns
				kick_defer_list_purge
				net_rps_send_ipi
			generic_exec_single ->

		smp_call_function_any
			smp_call_function_single
				generic_exec_single
					__smp_call_single_queue
						call_single_data_t (CSD)
						trace_cs_queue_cpu_enabled
						trace_csd_queue_cpu
						send_call_function_single_ipi ->
		smp_call_on_cpu
			smp_call_on_cpu_struct
			queue_work_on
			wait_for_completion

	cpu control
		isolation & housekeeping
		drain_all_stock
			!cpu_is_isolated
			schedule_work_on(cpu, &stock->work);
		vmstat_shepherd
			cpus_read_lock
			!cpu_is_isolated
			queue_delayed_work_on
				__queue_delayed_work

		schedule_work_on
			queue_work_on ->
		cpu_is_isolated
			!housekeeping_test_cpu(cpu, HK_TYPE_DOMAIN) ||
	       		!housekeeping_test_cpu(cpu, HK_TYPE_TICK) ||
			cpuset_cpu_is_isolated	include/linux/cpuset.h
				cpumask_test_cpu(cpu, isolated_cpus);

	on_each_cpu
		cpu_online_mask
		on_each_cpu_cond_mask
			smp_call_function_many_cond->


==== isolation

	/sys/devices/system/cpu/isolated:
		print_cpus_isolated
			< cpu_root_attrs
			housekeeping_cpumask(HK_TYPE_DOMAIN))


	kernel/sched/build_utility.c:104:#

	mv update_unbound_workqueue_cpumask update_isolation_cpumasks

	update_unbound_workqueue_cpumask
		workqueue_unbound_exclude_cpumask
			wq_isolated_cpumask = exclude_cpumask
			wq_requested_unbound_cpumask &= !exclude_cpumask
			workqueue_apply_unbound_cpumask
				workqueues
				apply_wqattrs_prepare
					alloc_workqueue_attrs
					alloc_unbound_pwq
					wqattrs_actualize_cpumask
						attrs->cpumask
					__pod_cpumask
				wq_unbound_cpumask

	update_isolation_cpumasks
		lockdep_assert_cpus_held
			percpu_rwsem_assert_held(&cpu_hotplug_lock);
		workqueue_unbound_exclude_cpumask ->
		housekeeping_exlude_isolcpus
			housekeeping_update
				blk_mq_flush_on_cpu
					blk_mq_hctx_notify_offline
					blk_mq_hctx_notify_dead
					bio_cpu_dead
					blk_softirq_cpu_dead
				blk_mq_hctx_notify_offline
				blk_mq_hctx_notify_dead
				bio_cpu_dead
				blk_softirq_cpu_dead
				- irq_migrate_all_off_cpu
				irq_affinity_adjust
					irq_restore_affinity_of_irq
					migrate_one_irq_from_isolated
					+__irq_can_set_affinity
					irq_set_affinity
					-irq_set_affinity_locked ->
		+kthreads_update_affinity
			set_cpus_allowed_ptr kthreadd_task
			kthreads_online_cpu->

	find_lowest_rq
		cpumask_any_distribute
			distribute_cpu_mask_prev
		cpumask_any_and_distribute
			< bpf_cpumask_any_and_distribute find_later_rq pick_idle_cpu_in_node scx_bpf_pick_any_cpu_node find_lowest_rq kick_pool
			distribute_cpu_mask_prev
			cpumask_next_and_wrap
				find_next_and_bit_wrap
					find_next_and_bit
						_find_next_and_bit
							FIND_NEXT_BIT


	sed 's/ /\n/g;s/=/\t/g' /proc/cmdline|column -t
	sed $'s/ /\\n/g;s/=/\e[40G/g' /proc/cmdline

		memcg_stat_show
			__mem_cgroup_flush_stats+75
			mem_cgroup_flush_stats
				do_flush_stats
					cgroup_rstat_flush
					cgroup_rstat_flush_irqsafe
		blkcg_print_blkgs

	long _raw_write_unlock_irq
		kernel_clone exit_notify
		run_timer_softirq

	do_notify_resume
	do_signal
	get_signal
	do_group_exit
	do_exit exit_notify
	_raw_write_unlock_irq
		__raw_write_unlock_irq
			do_raw_write_unlock
			local_irq_enable
				raw_local_irq_enable
					arch_local_irq_enable
			preempt_enable

	__bpf_kfunc

==== CPUHP...

	Call sequence: takedown_cpu, stop_machine_cpuslocked, multi_cpu_stop, local_irq_disable

	Sample challenge:
		smp_call_function -> smp_call_function_many_cond -> send_call_function_single_ipi, csd_lock_wait would wait forever 

		"smp_function_call, Thomas"
		send_call_function_ipi_mask
		on_each_cpu


	https://lore.kernel.org/lkml/87frmj73qj.ffs@tglx/

cpuhp online interferes oslat delay 3540 us RT; std 1300/5000
hwnoise - pass


echo 1 > $events/cpuhp/enable;
... grep 'cpuhp_.*enter'

cpuhp_hp_states

cpu offline:

	ap:
		cpuhp_kick_ap_work
			cpuhp_lock_acquire
			cpuhp_kick_ap
		CPUHP_AP_ACTIVE sched_cpu_deactivate
		mce_cpu_pre_down
		cpuid_device_destroy
		msr_device_destroy
		vmstat_cpu_down_prep
		kvm_cpu_down_prepare
		cacheinfo_cpu_pre_down
		CPUHP_AP_RCUTREE_ONLINE rcutree_offline_cpu
		workqueue_offline_cpu
		tmigr_cpu_offline
		rapl_cpu_offline
		perf_event_exit_cpu
		blk_mq_hctx_notify_offline	*
		smpboot_park_threads
		sched_cpu_wait_empty
		kvm_offline_cpu

	cpu 0:
		takedown_cpu ... &take_cpu_down
		CPUHP_BRINGUP_CPU finish_cpu
		crash_cpuhp_offline
		free_vm_stack_cache
		timers_dead_cpu
		topology_remove_dev
		rcutree_dead_cpu
		smpcfd_dead_cpu
		dev_cpu_dead
		page_alloc_cpu_dead
		radix_tree_cpu_dead
		percpu_counter_cpu_dead
		console_cpu_notify
		buffer_exit_cpu_dead
		blk_mq_hctx_notify_dead		*
		acpi_soft_cpu_dead
		bio_cpu_dead		*
		blk_softirq_cpu_dead	*
		takeover_tasklets
		vmstat_cpu_dead
		page_writeback_cpu_online
		slub_cpu_dead
		mce_cpu_dead
		x86_pmu_dead_cpu
		CPUHP_PERF_PREPARE = 2 perf_event_exit_cpu

cpuhp_hp_states

cpu online:
	cpu 0:
		CPUHP_CREATE_THREADS = 1 smpboot_create_threads
		CPUHP_PERF_PREPARE perf_event_init_cpu
		x86_pmu_prepare_cpu
		page_alloc_cpu_online
		random_prepare_cpu
		workqueue_prepare_cpu
		hrtimers_prepare_cpu
		smpcfd_prepare_cpu
		relay_prepare_cpu
		CPUHP_AP_RCUTREE_ONLINE rcutree_prepare_cpu
		topology_add_dev
		trace_rb_cpu_prepare
		trace_rb_cpu_prepare
		timers_prepare_cpu
		tmigr_cpu_prepare
		kvmclock_setup_percpu
		crash_cpuhp_online
		cpuhp_kick_ap_alive
		CPUHP_BRINGUP_CPU cpuhp_bringup_ap
			5 bringup_cpu
		CPUHP_AP_CACHECTRL_STARTING cache_ap_online
		...

	ap:
		kvm_online_cpu
		smpboot_unpark_threads
		irq_affinity_online_cpu
		blk_mq_hctx_notify_online
		perf_event_init_cpu
		x86_pmu_online_cpu
		rapl_cpu_online
		tmigr_cpu_online
		workqueue_online_cpu
		random_online_cpu
		rcutree_online_cpu
		cacheinfo_cpu_online
		kvm_cpu_online
		page_writeback_cpu_online
		vmstat_cpu_online
		kcompactd_cpu_online
		msr_device_create
		cpuid_device_create
		compute_batch_value
		acpi_soft_cpu_online
		mce_cpu_online
		CPUHP_AP_ONLINE_DYN console_cpu_notify
		CPUHP_AP_ACTIVE sched_cpu_activate

	? kthreads_online_cpu CPUHP_AP_KTHREADS_ONLINE

	irq_domain
	irq_matrix_debug_show
	head -n-1 /sys/kernel/debug/irq/domains/*

	kernel/stop_machine.c		- kernel/cpu.c
	cpu_stop_init
					- cpuhp_threads_init

	cpu_stop_threads
					- cpuhp_threads
			smpboot_thread_fn
	cpu_stopper_thread
	multi_cpu_stop
	kernel/cpu.c:
	   				- cpuhp_thread_fun
	take_cpu_down
	arch/x86/kernel/smpboot.c:
	native_cpu_disable
	cpu_disable_common
					- cpuhp_invoke_callback
	arch/x86/kernel/irq.c fixup_irqs
	kernel/irq/cpuhotplug.c:
	irq_migrate_all_off_this_cpu
					- irq_affinity_online_cpu
	migrate_one_irq
					- irq_restore_affinity_of_irq
	irqd_set_managed_shutdown
					- irq_startup

	cpuhp_up_callbacks
		cpuhp_invoke_callback
			smpboot_create_threads
				__smpboot_create_thread
			perf_event_init_cpu
			x86_pmu_prepare_cpu
			page_alloc_cpu_online
			random_prepare_cpu
			workqueue_prepare_cpu
			hrtimers_prepare_cpu
			smpcfd_prepare_cpu
			relay_prepare_cpu
			rcutree_prepare_cpu
				rcu_node
				//rcu_spawn_rnp_kthreads
					rcu_spawn_one_boost_kthread
						"rcub"
						@rcu_boost_kthread
							rcu_boost
						rcu_thread_affine_rnp ->
					rcu_spawn_exp_par_gp_kworker
						kthread_create_worker
						"rcu_exp_par_gp_kthread_worker"
						rcu_thread_affine_rnp
							for_each_leaf_node_possible_cpu
								cpu_possible_mask
								cpumask_next
								grplo
								grphi
							kthread_affine_preferred ->
			topology_add_dev
			trace_rb_cpu_prepare
			timers_prepare_cpu
			kvmclock_setup_percpu
			crash_cpuhp_online
			cpuhp_kick_ap_alive
				arch_cpuhp_kick_ap_alive
					kick_ap_alive -> native_kick_ap
						do_boot_cpu
							initial_code = start_secondary
			cpuhp_bringup_ap
				cpuhp_kick_ap
			cache_ap_online
				TODO
				&cache_rendezvous_handler ->
				stop_machine_from_inactive_cpu
					while (!mutex_trylock(&stop_cpus_mutex))
					cpu_active_mask
					queue_stop_cpus_work ->
					multi_cpu_stop ->
			sched_cpu_starting
				sched_core_cpu_starting
					cpu_smt_mask
				sched_rq_cpu_starting
				sched_tick_start ->
			x86_pmu_starting_cpu
				intel_pmu_cpu_starting
			kvm_online_cpu
				__hardware_enable_nolock
			smpboot_unpark_threads
				smpboot_unpark_thread
					selfparking
					kthread_unpark ->
			irq_affinity_online_cpu
			...

	cpuhp_down_callbacks...
		cpuhp_invoke_callback_range ...

		cpuhp_invoke_callback ->
			rcutree_offline_cpu
			workqueue_offline_cpu
			tmigr_cpu_offline
			rapl_cpu_offline
			perf_event_exit_cpu
			blk_mq_hctx_notify_offline ->
			smpboot_park_threads
				smpboot_park_thread
					selfparking
					kthread_park
			sched_cpu_wait_empty
				balance_hotplug_wait
			kvm_offline_cpu
			takedown_cpu ->
			tick_cpu_dying -- stops tick_nohz_handler
				tick_do_timer_cpu - time keeper cpu
				tick_sched_timer_dying
					tick_sched_timer_cancel
						hrtimer_cancel ->
				tick_offline_cpu
					tick_broadcast_offline(cpu);
						tick_broadcast_oneshot_offline(cpu);
						tick_shutdown_broadcast
							clockevents_shutdown
								clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
			hrtimers_cpu_dying
				bpftrace -e 'kprobe:hrtimers_cpu_dying { @[kstack()] = count(); }'
			smpcfd_dying_cpu
			x86_pmu_dying_cpu
			rcutree_dying_cpu
			sched_cpu_dying
				sched_tick_stop
			cache_ap_offline


		target_store
			cpuhp_get_step
			cpu_up ->
			cpu_down
				cpu_maps_update_begin
					mutex_lock
						cpu_add_remove_lock
							Serializes the updates to cpu_online_mask, cpu_present_mask
				cpu_down_maps_locked
					work_on_cpu
						work_on_cpu_key
							schedule_work_on ->
							flush_work
								__flush_work
									start_flush_work
										rcu_read_lock
										raw_spin_lock_irq
										raw_spin_unlock_irq
									wait_for_completion
							destroy_work_on_stack
					__cpu_down_maps_locked
						_cpu_down
							cpus_write_lock
								percpu_down_write
								cpu_hotplug_lock
							cpuhp_down_callbacks
								cpuhp_invoke_callback_range
									__cpuhp_invoke_callback_range
										cpuhp_next_state
										cpuhp_invoke_callback
											timers_dead_cpu ->
				cpu_maps_update_done
					mutex_unlock
					cpu_add_remove_lock

	take_cpu_down
	__cpu_disable
	smp_ops.cpu_disable
	native_cpu_disable
		lapic_can_unplug_cpu
			raw_spin_lock(&vector_lock);
		cpu_disable_common
			remove_cpu_from_maps
				set_cpu_online
					__cpu_online_mask -> cpu_online_mask
					__num_online_cpus -> num_online_cpus
				numa_remove_cpu
					node_to_cpumask_map
			fixup_irqs ->


	hotplug
		device_online
		device_offline
		/sys/devices/system/cpu/hotplug/states
	hotplug disturbance
		
		cd /sys/devices/system/cpu/; i=20; echo 0 > cpu$i/online; sleep 1; echo 1 > cpu$i/online; sleep 1;
		oslat -c 11  -D 10

		rtla hwnoise 0
		rtla osnoise 2700
		oslat 2700
		rtla timerlat hist -c 10 2500
		cyclictest 1800

	rtla osnoise hist -d 5m -c 8 -Pf:$p

	ret_from_fork_asm
	ret_from_fork
		kthread
			smpboot_thread_fn
				smpboot_thread_data
					smp_hotplug_thread
				cpu_stopper_thread  ->
				setup
				park
				unpark
				thread_fn

	...
				cpu_stopper_thread < cpu_stop_threads
					- runs all the time
					preempt_count_inc ..
					cpu_stopper
					local_irq_disable ->
					hard_irq_disable
					multi_cpu_stop
						unsigned long flags;
						local_save_flags
							raw_local_save_flags
								arch_local_save_flags
						local_irq_disable ->
						hard_irq_disable();
						fn()
						on cpu offline:
							take_cpu_down
						on online:
							cache_rendezvous_handler
								get_cache_aps_delayed_init
								cache_cpu_init
									local_irq_save(flags);
									cache_disable
									mtrr_generic_set_state
										MTRR - Memory Type Range Register
									pat_cpu_init
						local_irq_restore(flags);
					ack_state
						MULTI_STOP_PREPARE -> MULTI_STOP_DISABLE_IRQ -> MULTI_STOP_RUN
						atomic_dec_and_test(thread_ack)
						set_state +1
							thread_ack = num_threads
					rcu_momentary_eqs
					cpu_stop_signal_done

	CONFIG_HOTPLUG_CPU
			unregister_cpu
			arch_cpu_probe
			arch_cpu_release
			remove_cpu ...


	tests
	CONFIG_TORTURE_TEST torture
			torture_init_begin
				torture_type
				torture_print_module_parms
			torture_onoff_init kernel/torture.c
				torture_onoff
					torture_hrtimeout_jiffies
					torture_must_stop
						torture_must_stop_irq
					torture_online
						add_cpu
							device_online ->
								...
									cpu_subsys_online ->
					torture_offline
						remove_cpu
							device_offline
								...
									cpu_subsys_offline ->
			torture_shutdown_init
				torture_shutdown_hook
			torture_cleanup_begin
				WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
			torture_cleanup_end




	CONFIG_RCU_SCALE_TEST rcuscale
		rcu_scale_init ./kernel/rcu/rcuscale.c:815
			torture_init_begin

	CONFIG_RCU_TORTURE_TEST
		rcutorture
			rcu_torture_init
				torture_init_begin
				torture_onoff_init

	CONFIG_RCU_REF_SCALE_TEST refscale
		ref_scale_init ./kernel/rcu/refscale.c
			torture_init_begin


	CONFIG_SCF_TORTURE_TEST scftorture
		scf_torture_init
			scftorture_invoker
			torture_init_begin
			onoff_interval
			torture_onoff_init->
			...
			scftorture_invoke_one
				smp_call_function ->

	CONFIG_LOCK_TORTURE_TEST
	lock_torture_init kernel/locking/locktorture.c
		torture_init_begin
		onoff_interval
		torture_onoff_init kernel/torture.c
		torture_shutdown_init
	lock_torture_cleanup
		torture_cleanup_begin
		torture_cleanup_end

	kunit
	kcsan_test_suite
		test_init ./kernel/kcsan/kcsan_test.c:1091
			torture_init_begin


		cpu_subsys
			cpu_subsys_online
				cpu_device_up(dev);
					cpu_up(dev->id, CPUHP_ONLINE);
						cpu_maps_update_begin ->
						try_online_node(cpu_to_node(cpu));
							mem_hotplug_begin
								cpus_read_lock

						_cpu_up(cpu, 0, target);
							cpus_write_lock
								percpu_down_write(&cpu_hotplug_lock);
									__percpu_down_write_trylock
							cpuhp_up_callbacks
								cpuhp_invoke_callback_range
									__cpuhp_invoke_callback_range ->
										cpuhp_invoke_callback ->
						cpu_maps_update_done ->
			cpu_subsys_offline
				cpu_device_down(dev)
					cpu_down(dev->id, CPUHP_OFFLINE);
			hotpluggable
			sched_cpu_wait_empty
			sched_cpu_dying
			idle_task_exit
			cpuhp_hp_states
				CPUHP_HRTIMERS_PREPARE
					hrtimers_prepare_cpu
						timerqueue_init_head
					hrtimers_cpu_dying | hrtimers_dead_cpu ->
				CPUHP_TIMERS_PREPARE
					timers_prepare_cpu ->
					timers_dead_cpu ->
			rcutree_dead_cpu
				tick_dep_clear(TICK_DEP_BIT_RCU);
			rcutree_dying_cpu
			rcutree_offline_cpu
				tick_dep_set(TICK_DEP_BIT_RCU);
			rcu_lockdep_current_cpu_online
				rcu_rdp_cpu_online
					rcu_rnp_online_cpus
			torture_num_online_cpus
			tick_broadcast_offline

			cpuhp_hp_states
				cpuhp_bringup_ap
					bringup_wait_for_ap_online
						wait_for_ap_thread
							done_up
							done_down
						kthread_unpark ->

					cpuhp_kick_ap
						__cpuhp_kick_ap
							should_run
							wake_up_process
							wait_for_ap_thread

				sched_cpu_activate
					cpuset_cpu_active
					- cpuset_update_active_cpus
					- - partition_sched_domains ->
					- - cpuset_handle_hotplug
						lockdep_assert_cpus_held (cpu_hotplug_lock)
						mutex_lock(&cpuset_mutex)
						cpuset_hotplug_update_tasks
							mutex_lock(&cpuset_mutex);
							remote_partition_disable
							hotplug_update_tasks
								cpuset_update_tasks_cpumask ->
								cpuset_update_tasks_nodemask
									cpuset_change_task_nodemask
					- cpuset_reset_sched_domains 
					- - partition_sched_domains ->
				sched_cpu_deactivate
					set_cpu_active

				"irq/affinity:online"
				irq_affinity_online_cpu
				CPUHP_AP_HRTIMERS_DYING
				hrtimers_cpu_dying | hrtimers_dead_cpu
					hrtimer_bases
						hrtimer_cpu_base ->
					cpu_active_mask
					raw_spin_lock
					raw_spin_lock_nested
					x tick_cancel_sched_timer
						sched_timer
						hrtimer_cancel ->
					migrate_hrtimer_list
						timerqueue_node
						timerqueue_getnext
							timerqueue_head
							rb_first_cached
						__remove_hrtimer
						enqueue_hrtimer ->
							timerqueue_add
					__hrtimer_get_next_event
						__hrtimer_next_event_base
					smp_call_function_single ->
			tick_handover_do_timer
			tick_shutdown

			takeover_tasklets

			cpuhp_setup_state_nocalls
				__cpuhp_setup_state
					cpus_read_lock
						cpu_hotplug_lock
						percpu_down_read
					__cpuhp_setup_state_cpuslocked
						cpuhp_store_callbacks
						cpuhp_issue_call ->
					cpuhp_invoke_callback
						cpuhp_get_step
						hrtimers_cpu_dying ->

			object_cpu_offline
			cpuhp_issue_call
				cpuhp_invoke_ap_callback
					cpuhp_lock_acquire
					__cpuhp_kick_ap ->
				cpuhp_invoke_callback ->

			smpboot_thread_fn
				cpuhp_threads.thread_fn
					cpuhp_thread_fun
						lockdep_acquire_cpus_lock
							rwsem_acquire
								lock_acquire_exclusive
									lock_acquire ->
							cpu_hotplug_lock.dep_map
						cpuhp_lock_acquire
						cpuhp_state
						local_irq_disable
						cpuhp_invoke_callback
							sched_cpu_activate
								set_cpu_active

stop_machine* usages

	takedown_cpu take_cpu_down

	do_optimize_kprobes

	include/linux/ftrace.h
	arch/arm/kernel/ftrace.c
		arch_ftrace_update_code	__ftrace_modify_code
	arch/arm/kernel/patch.c
		patch_text	patch_text_stop_machine

	arch/arm/mm/init.c
		fix_kernmem_perms	__fix_kernmem_perms
		mark_rodata_ro	__mark_rodata_ro
		early_fixmap_init
	arch/arm/probes/kprobes/core.c
		kprobes_remove_breakpoint __kprobes_remove_breakpoint
		kprobes_remove_breakpoint __kprobes_remove_breakpoint


	Srivatsa S. Bhat:
		_cpu_down:
			__stop_machine -> stop_one_cpu
		+
			include/linux/percpu-rwlock.h
			lib/percpu-rwlock.c
				percpu_rwlock
		hotplug_pcpu_rwlock
		get_online_cpus_atomic
			percpu_read_lock_irqsafe(&hotplug_pcpu_rwlock);
		put_online_cpus_atomic
			percpu_read_unlock_irqsafe(&hotplug_pcpu_rwlock);
		
		change patterns:

		Subject: [PATCH v6 10/46] smp, cpu hotplug: Fix smp_call_function_*() to prevent CPU offline properly
		-       this_cpu = get_cpu();
		+       get_online_cpus_atomic();
		+       this_cpu = smp_processor_id();

		-       put_cpu();
		+       put_online_cpus_atomic();

		percpu_rwlock:
			01 Introduce the global reader-writer lock backend
			02 Introduce per-CPU variables for the reader and the writer
			03 Provide a way to define and init percpu-rwlocks at compile time
			04 Implement the core design of Per-CPU Reader-Writer Locks
			05 Make percpu-rwlocks IRQ-safe, optimally
			06 Rearrange the read-lock code to fastpath nested percpu readers
			07 Allow writers to be readers, and add lockdep annotations

		Assorted:
			08 CPU hotplug: Provide APIs to prevent CPU offline from atomic context
			09 CPU hotplug: Convert preprocessor macros to static inline functions
			10 smp, cpu hotplug: Fix smp_call_function_*() to prevent CPU offline properly
			11 smp, cpu hotplug: Fix on_each_cpu_*() to prevent CPU offline properly

			13 sched/migration: Use raw_spin_lock/unlock since interrupts are already disabled
			17 clockevents: Use get/put_online_cpus_atomic() in clockevents_notify()
			22 crypto: pcrypt - Protect access to cpu_online_mask with get/put_online_cpus()

		Use get/put_online_cpus_atomic() to prevent CPU offline:
			kernel:
				12 sched/timer
				14 sched/rt
				15 tick
				16 time/clocksource
				18 softirq
				19 irq
			root:
				20 net
				21 block
			drivers:
				23 infiniband, ehca
				24 [SCSI] fcoe
				25 staging, octeon
			arch:
				26 x86
				27 perf/x86
				28 KVM ... from atomic context
				29 kvm/vmx
				30 x86/xen
				31 alpha/smp
				32 blackfin/smp
				33 cris/smp
				34 hexagon/smp
				35 ia64
				36 m32r
				37 MIPS
				38 mn10300
				39 parisc
				40 powerpc
				41 sh
				42 sparc
				43 tile

		stop_machine:
			45 CPU hotplug, stop_machine: Decouple CPU hotplug from stop_machine() in Kconfig
			46 Documentation/cpu-hotplug: Remove references to stop_machine()

		lglock: add read-preference local-global rwlock

		generated by:
		pcregrep  -M 'Subject. \[.*\n\s' a.txt | sed '/Subject/{ N; s/\n / / }'| sort -u 




=== cpumask
	mm/percpu.c

		per_cpu_ptr
			per_cpu_offset
				__per_cpu_offset

	include/linux/cpumask.h
		alloc_cpumask_var
		cpu_online
			cpu_online_mask
		cpumask_and
			bitmap_and
		cpumask_clear
			bitmap_zero
		cpumask_clear_cpu
		cpumask_copy
		cpumask_empty
		cpumask_first
		cpumask_set_cpu
		cpumask_test_cpu
		cpumask_weight
		free_cpumask_var
		num_online_cpus
		zalloc_cpumask_var
			alloc_cpumask_var

		cpumask
			DECLARE_BITMAP
		cpumask_weight
			cpumask_bits
		cpumask_weight_and
			cpumask_bits
				cpumask.bits
		cpumask_any_and
			cpumask_first_and
				find_first_and_bit
					_find_first_and_bit
						IND_FIRST_BIT
							for
							__ffs
								bsf - Bit Scan Forward
							break
		cpumask_intersects
			bitmap_intersects
				__bitmap_intersects
					for
					&
					return

=== Other system

==== kernel/

		kernel/acct.c
			sys_acct
				acct_on
					bsd_acct_struct
						acct_file_reopen

		kernel/capability.c
			sys_capget
				kernel_cap_t
				cap_get_target_pid
			sys_capset lkm2
				cred
				security_capset lkm2
				prepare_creds
				commit_creds

		kernel/exec_domain.c
			sys_personality

		kernel/exit.c
			sys_exit_group
				do_group_exit
			sys_exit
				do_exit
					exit_mm
						mm_release
					exit_sem
					exit_files
					exit_fs
					check_stack_usage
					exit_thread
					cgroup_exit
					exit_notify
						tasklist_lock
						write_lock_irq
						forget_original_parent
						release_task
							write_lock_irq
			sys_wait4
				do_wait
					do_wait_thread

			sys_waitid
				do_wait

		kernel/fork.c
			sys_set_tid_address
				current->clear_child_tid = tidptr;
			sys_unshare
			sys_fork  ▻

		kernel/futex.c
			futex_init
				futex_queues
			sys_futex	lkm2
				do_futex
					futex_wait
					futex_wake
			sys_get_robust_list
				current->robust_list
			sys_set_robust_list
				current->robust_list

			sys_getgroups		kernel/groups.c
				current_cred
				cred->group_info
			sys_setgroups		kernel/groups.c


		kernel/kexec.c
			sys_kexec_load
				kimage_normal_alloc
					kimage_alloc_control_pages
						kimage_alloc_normal_control_pages
							kimage_alloc_pages
				kimage_crash_alloc
					kimage_alloc_control_pages
						kimage_alloc_crash_control_pages

		kernel/module.c
			sys_delete_module  ▻
			sys_init_module  ▻

		include/linux/perf_event.h
			perf_pmu_register
			perf_event_read_value

		kernel/perf_event.c
			sys_perf_event_open perf_event_open
				perf_event_attr
				perf_event
				perf_event_alloc
				anon_inode_getfd  ▻
				current->perf_event_list
			perf_event_init <- sched_init
			init_hw_perf_events	x86/kernel/cpu/perf_event.c
				intel_pmu_init
		kernel/printk.c
			sys_syslog  ▻

		kernel/ptrace.c
			sys_ptrace  ▻ lkm2
				lock_kernel
				arch_ptrace
					ptrace_request
						ptrace_resume

		kernel/sys.c
			sys_getpgid
				find_task_by_vpid  ▻
				task_pgrp
					task->group_leader->pids[PIDTYPE_PGID].pid
			sys_getpgrp
				sys_getpgid
			sys_getpriority
				task_nice
					PRIO_TO_NICE((p)->static_prio);
			sys_getresgid
				cred = current_cred
			sys_getresuid
				cred = current_cred
			sys_getrlimit
				current->signal->rlim
			sys_getrusage
				k_getrusage
			sys_getsid
			sys_prctl
				perf_event_task_enable
					perf_event_enable
			sys_reboot  ▻
			sys_setdomainname
				utsname
				domainname
			sys_setfsgid
			sys_setfsuid
			sys_setgid
			sys_sethostname
				utsname
				nodename
			sys_setpgid
				task_pid_vnr
					task_pgrp
					change_pid
			sys_setpriority
				set_one_prio
			sys_setregid
			sys_setresgid
			sys_setresuid
			sys_setreuid
			sys_setrlimit
			sys_setsid
			sys_setuid
			sys_times  ▻
			sys_umask
				current->fs->umask

		kernel/sysctl_binary.c
			sys_sysctl
				do_sysctl  ▻

		kernel/time.c
			sys_adjtimex
				do_adjtimex
			sys_gettimeofday  ▻
			sys_settimeofday
				do_sys_settimeofday
					do_settimeofday
			sys_time  ▻

		kernel/timer.c
			sys_alarm
				alarm_setitimer
					do_setitimer  ▻
			sys_getegid
			sys_geteuid
			sys_getgid
			sys_getpid
			sys_getppid
			sys_gettid
			sys_getuid
			sys_sysinfo lkm2
				do_sysinfo
					si_meminfo
						totalram_pages	lkm2
						totalhigh_pages
						global_page_state(NR_FREE_PAGES)
					si_swapinfo
						swap_info lkm2
						total_swap_pages
		tracer_alloc_buffers
		tracer_init_debugfs

===== RCU

	update_process_times
		rcu_sched_clock_irq
			trace_rcu_utilization(TPS("Start scheduler-tick"));
			invoke_rcu_core
				use_softirq
				raise_softirq(RCU_SOFTIRQ
				invoke_rcu_core_kthread RT
					rcu_data
					rcu_cpu_kthread_status
					rcu_cpu_kthread_task
					rcu_wake_cond
						wake_up_process
			rcu_note_voluntary_context_switch
			trace_rcu_utilization(TPS("End scheduler-tick"));


include/linux/rcupdate.h

	call_rcu
	kfree_rcu
	rcu_dereference_protected
	list_for_each_entry_rcu
	struct rcu_head

	writer:
		rcu_assign_pointer
			smp_store_release
				WRITE_ONCE(*p, v)
		synchronize_rcu
	reader:
		rcu_read_lock
			__rcu_read_lock
				preempt_disable
			__acquire(RCU); sparse annotation
			rcu_lock_map
			cu_lock_acquire CONFIG_DEBUG_LOCK_ALLOC
				lock_acquire
		rcu_dereference
			rcu_dereference_check
				__rcu_dereference_check
					rcu_check_sparse
		rcu_read_unlock
			rcu_lock_release CONFIG_DEBUG_LOCK_ALLOC


kernel/rcu/tiny.c

	rcu_process_callbacks
		rcu_ctrlblk
		rcu_reclaim_tiny
			rcu_head
			func

include/linux/rcutree.h

kernel/rcu/tree.h

	rcu_node


Noise
	security_inode_free
		call_rcu
		inode_free_by_rcu

	release_probes
		rcu_free_old_probes

rcuc/8-103     [008] b...2 17162.478731: rcu_invoke_callback: rcu_preempt rhp=00000000775cdca1 func=rcu_free_old_probes


kernel/rcu/tree.c

	rcutree_enqueue
		rcu_callback
		inode_free_by_rcu

rtla-3287    [008] d....   331.872003: rcu_callback: rcu_preempt rhp=000000003ecbd778 func=inode_free_by_rcu 17
		rhp: Pointer to the RCU head


        rcu_accelerate_cbs
		rcu_segcb_stats

rtla-3287    [008] d....   331.872003: rcu_segcb_stats: SegCBQueued seglen: (DONE=0, WAIT=0, NEXT_READY=0, NEXT=17) gp_seq: (DONE=0, WAIT=20628, NEXT_READY=20628, NEXT=0)


	rcu_scheduler_starting

	call_rcu
		__call_rcu_common
			call_rcu_nocb
				rcutree_enqueue
					cblist
			call_rcu_core
				rcutree_enqueue

	synchronize_rcu
		synchronize_rcu_normal

	rcu_thread_affine_rnp
		for_each_leaf_node_possible_cpu
		zalloc_cpumask_var
		kthread_affine_preferred ->

	CONFIG_HOTPLUG_CPU
		takedown_cpu
			rcutree_migrate_callbacks
				rcu_rdp_is_offloaded
				cblist
				rcu_advance_cbs
				rcu_segcblist_merge
		rcutree_dead_cpu

	rcu_get_n_cbs_cpu
		cblist

	rcu_spawn_gp_kthread
		rcu_spawn_core_kthreads
			smpboot_register_percpu_thread
			rcu_cpu_thread_spec
				rcu_cpu_kthread "rcuc"
					rcu_core
						cblist
						rcu_do_batch
							cblist
							func()
							trace_rcu_invoke_callback
						do_nocb_deferred_wakeup

	TREE_RCU


	RCU_NOCB_CPU



kernel/rcu/tree_nocb.h

	rcu_nocb_setup
		rcu_nocb_mask
		"rcu_nocbs"

	CONFIG_RCU_NOCB_CPU

		rcu_init_nohz
			tick_nohz_full_mask
			rcu_organize_nocb_kthreads
			rcu_nocb_mask
		rcu_torture_init
			rcu_nocb_toggle
				rcu_nocb_cpu_offload
					rcu_rdp_is_offloaded
						rcu_segcblist_is_offloaded
							rcu_segcblist_test_flags
							SEGCBLIST_LOCKING
					rcu_nocb_mask
					work_on_cpu
					&rcu_nocb_rdp_offload
						rcu_nocb_queue_toggle_rdp
							nocb_toggling_rdp
						?rdp_offload_toggle
				rcu_nocb_cpu_deoffload
					rcu_rdp_is_offloaded
					rcu_nocb_rdp_deoffload
						?rdp_offload_toggle
							?rcu_segcblist_offload
		rcu_nocb_cb_kthread


==== cgroup

	cgroup_path_ns
		cgroup_lock
			cgroup_mutex - master lock

		spin_lock_irq
		css_set_lock


	task_struct.cgroups
	css_set
	css_get / css_put

===== cpuset

	rebuild_sched_domains
		cpus_read_lock
		rebuild_sched_domains_cpuslocked
			mutex_lock(&cpuset_mutex);
			rebuild_sched_domains_locked
				lockdep_assert_cpus_held
				lockdep_assert_held(&cpuset_mutex);
				generate_sched_domains
					alloc_sched_domains
					top_cpuset.effective_cpus
					HK_TYPE_DOMAIN
					update_domain_attr_tree
						is_sched_load_balance
						update_domain_attr
				partition_sched_domains
					partition_sched_domains_locked
						arch_update_cpu_topology
						asym_cpu_capacity_scan
							asym_cpu_capacity_update_data
						alloc_sched_domains
				    		housekeeping_cpumask(HK_TYPE_DOMAIN));
						build_sched_domains
							build_sched_domain
								sched_domain_span
									to_cpumask
								set_domain_attribute
							cpu_attach_domain ->
						doms_cur
			mutex_unlock(&cpuset_mutex);
		cpus_read_unlock


	

	cpuset_hotplug_workfn
		rebuild_sched_domains

	update_partition_exclusive
		CS_CPU_EXCLUSIVE

	cpu_exclusive is_cpu_exclusive CS_CPU_EXCLUSIVE
	sched_load_balance is_sched_load_balance

include/linux/cpuset.h

	cpuset_lock
		mutex_lock
		cpuset_mutex

Explanantion comment about cpuset_mutex (after top_cpuset):

	There are two global locks guarding cpuset structures:
  		cpuset_mutex and
  		callback_lock.


	cpuset and hotplug
		hotplug_update_tasks
		cpuset_handle_hotplug

	cpuset1_hotplug_update_tasks
		cpuset_callback_lock_irq
			spin_lock_irq
			callback_lock - spinlock

===== rstat

	kubelet
	ksys_read
	vfs_read
	kernfs_fop_read_iter
	seq_read_iter
	kernfs_seq_show
	cgroup_seqfile_show
	memory_stat_show < memory_files
		taskset --cpu-list 0 cat /sys/fs/cgroup/memory.stat > /dev/nul
		memory_stat_format.constprop.0
			mem_cgroup_flush_stats
				should_flush_stats
				do_flush_stats
					cgroup_rstat_flush __bpf_kfunc 200+ us
						cgroup_rstat_lock
						spin_lock_irq
							raw_spin_lock_irq ->
						cgroup_rstat_flush_locked 200+us ->
							cgroup_rstat_updated_list 0-20 us ->
							need_resched
						spin_unlock_irq
							raw_spin_unlock_irq
								_raw_spin_unlock_irq
									__raw_spin_unlock_irq
                                                                                do_raw_spin_unlock
                                                                                        queued_spin_unlock
												pv_queued_spin_unlock
												smp_store_release
													WRITE_ONCE
                                                                                local_irq_enable
                                                                                preempt_enable

	cgroup_account_cputime
		__cgroup_account_cputime
			cgroup_base_stat_cputime_account_end
				cgroup_rstat_updated

	cgroup_base_stat_cputime_show
		cgroup_rstat_flush_hold
		cgroup_rstat_flush_release

	 _raw_spin_unlock_irq
	    el1h_64_irq
		...
			try_to_wake_up

	kworker/u256:6
		mem_cgroup_flush_stats+132

	process_one_work
	wb_workfn
	wb_do_writeback
	wb_writeback
	spin_lock(&wb->list_lock);
	wb_over_bg_thresh
		mem_cgroup_wb_stats
			mem_cgroup_flush_stats_atomic
				do_flush_stats
					cgroup_rstat_flush_atomic 200+ us
						spin_lock_irqsave
						cgroup_rstat_flush_locked ->
	_raw_spin_unlock_irqrestore

	copy-content

	ksys_write
	vfs_write
	xfs_file_write_iter
	xfs_file_buffered_write
	iomap_file_buffered_write
	iomap_write_iter
	balance_dirty_pages_ratelimited_flags
	balance_dirty_pages
	mem_cgroup_wb_stats
		mem_cgroup_flush_stats_atomic
		6 mem_cgroup_flush_stats_ratelimited
			mem_cgroup_flush_stats
	cgroup_rstat_flush_atomic

	cgroup_rstat_updated_list:
	The _irqsave() is needed because cgroup_rstat_lock is spinlock_t which is a sleeping lock on PREEMPT_RT.
	Acquiring this lock with the _irq() suffix only disables interrupts on a non-PREEMPT_RT kernel.
	The raw_spinlock_t below disables interrupts on both configurations.
	The _irqsave() ensures that interrupts are always disabled and later restored.

		kernel/cgroup/rstat.c
		cgroup_rstat_flush_locked
			cgroup_rstat_updated_list longman, manager James Mills
				cgroup_rstat_cpu_lock
				raw_spin_lock_irqsave(cpu_lock, flags);
                                        _raw_spin_lock_irqsave
                                        do_raw_spin_lock

				cgroup_rstat_push_children
					cgroup_rstat_cpu
			/* if @may_sleep, play nice and yield if necessary */
			if (may_sleep && (need_resched() ||
					  spin_needbreak(&cgroup_rstat_lock))) {
				spin_unlock_irq(&cgroup_rstat_lock);
				if (!cond_resched())
					cpu_relax();
				spin_lock_irq(&cgroup_rstat_lock);
			}

	"cpu_dma_latency"
		cpu_latency_qos_write
			cpu_latency_qos_update_request
				cpu_latency_qos_apply(req, PM_QOS_UPDATE_REQ, new_value);
					pm_qos_update_target
						plist_node_init
						plist_add
						pm_qos_set_value
							target_value
					wake_up_all_idle_cpus

		apply_constraint
			pm_qos_read_value
			set_latency_tolerance
				?acpi_lpss_set_ltr

	Max Latencies: 08124

	echo 2 > /proc/irq/24/smp_affinity_list; cat /proc/interrupts; dmesg -c

	mlx5e_open_channel
		...
		netif_set_xps_queue
	mlx5_alloc_irq_vectors

	power
		dnf install -yq kernel-tools
		turbostat
		cpupower
		grep '' -r /sys/devices/system/cpu/cpu0/cpuidle/


		/sys/devices/system/cpu/cpu9/cpuidle/state3/latency

		cpuidle_state
		cpuidle_state_usage

		online irq_affinity_online_cpu

==== Sys APIs

===== sysfs
		attribute_group
			attribute
				name mode
		API
			sample usage
				module_notes_attrs
					bin_attribute
					module_notes_read
				add_notes_attrs
					sysfs_bin_attr_init
					sysfs_create_bin_file
				kobject_init_and_add
				mod_sysfs_setup
					module_add_modinfo_attrs
						sysfs_attr_init
						sysfs_create_file  ▻
				example_init
			kobject_create_and_add
				kobject_create
					kzalloc
					dynamic_kobj_ktype
					kobject_init
						kobject_init_internal
				kobject_add  ▻
			kobject_init_and_add
				kobject_init  ▻
				kobject_add_varg  ▻
			bin_attribute
				attribute
			sysfs_bin_attr_init
			sysfs_attr_init
			sysfs_create_dir - not used
			sysfs_create_file
				sysfs_add_file
					sysfs_add_file_mode
					x sysfs_make_dirent
						sysfs_new_dirent
			kobj_attribute (kobject.h)
		sysfs_init
			sysfs_fs_type
				sysfs_get_sb
					sysfs_fill_super
						sysfs_ops
					get_sb_single
			register_filesystem
		sysfs_create
		sysfs_ops

===== proc_fs

		fs/proc

		/proc/stat kstat_read_proc kstat

		proc_cpuinfo_operations

		CONFIG_PROC_FS
		    proc_root_init
			do_rw_proc

		sysctl_init
			register_proc_table
				proc_sys_file_operations
					proc_fs  ▻
			init_irq_proc

===== modules

		include/linux/module.h
		kernel/module.c

			struct module (module_state)
				kernel_param lkm2

			x sys_create_module
			sys_init_module
				load_module
					find_sec
					find_module
					module_alloc_update_bounds
						module_alloc
					mod_sysfs_init
					mod_sysfs_setup
				do_one_initcall
			sys_delete_module
			sys_query_module
			sys_get_kernel_syms

		2.4
			create_module()
			init_module()
			delete_module()

			insmod
			- module.c module_init() <include/linux/init.h>
			- module_exit() <include/linux/init.h>

		busybox
			insmod_main
				query_module
				new_get_kernel_symbols
				add_kernel_symbols
				create_module

		kernel_module module_list

			init_modules

		MOD_INC_USE_COUNT/MOD_DEC_USE_COUNT <include/linux/module.h>
		init_module()
		cleanup_module()
		EXPORT_SYMBOL()
		EXPORT_SYMTAB

		Chapter 11
		kmod and Advanced Modularization
		Contents:
		Loading Modules on Demand
		Intermodule Communication
		Version Control in Modules

		Chapter 2
		Building and Running Modules
		Contents:
		Kernel Modules Versus Applications
		Compiling and Loading
		The Kernel Symbol Table
		Initialization and Shutdown
		Using Resources
		Automatic and Manual Configuration
		Doing It in User Space

		/etc/modules, /etc/conf.modules

===== sysctl

		sysctl.h: General linux system control interface
		/linux/include/linux/sysctl.h  _LINUX_SYSCTL_H
		/linux/kernel/sysctl.c
			do_sysctl
			proc_sys_file_operations
				proc_fs  ▻

	system_calls	#337

		sys_syscall
		sys_call_table 	syscall_table_32.S
		arch/x86/kernel/syscall_64.c
			./source/arch/x86/include/asm/unistd_64.h
		ia32_sys_call_table	ia32entry.S

		__vectors_start

		arm
			vector_table
			vector_swi
				arm_syscall
					ptrace_break
					__show_regs
						show_regs_common
							dmi_get_system_info(DMI_SYS_VENDOR);
							dmi_get_system_info(DMI_PRODUCT_NAME)
							dmi_get_system_info(DMI_BOARD_NAME);
					c_backtrace
				__irq_entry
					IPSR	Interrupt Program Status Register
					irq = ipsr -16
					asm_do_IRQ
						handle_IRQ
							generic_handle_irq
									irq_to_desc
								generic_handle_irq_desc
									desc->handle_irq

		???
		x system_call (arch/i386/kernel/entry.S)
			sys_call_table (arch/i386/kernel/entry.S)
				sys_open  ▻

					vfs_read
						__vfs_read
							file->f_op->read
							new_sync_read
								filp->f_op->read_iter
							do_readv_writev  ▻
						...
						tty_read  ▻
						x do_sync_read
							generic_file_aio_read .aio_read
							sock_aio_read .aio_read
								sock->ops->recvmsg
									sock_common_recvmsg ssh ?
										tcp_recvmsg
											skb_copy_datagram_iovec
												memcpy_toiovec
				sys_write
				sys_socketcall __NR_socketcall

		include/asm/uaccess.h
			copy_from_user
			copy_to_user

		9. System Calls
		      POSIX APIs and System Calls
		      System Call Handler and Service Routines
		      Kernel Wrapper Routines

		sys_mmap2
		sys_mmap	lkm2
			ksys_mmap_pgoff
			vm_mmap_pgoff
				do_mmap
			do_mmap_pgoff /mm/mmap.c
				get_unmapped_area
					get_area
						arch_get_unmapped_area_topdown
							find_vma
				ERR find_vma_prepare
				may_expand_vm
				mm_struct
				kmem_cache_alloc  ▻
				security_file_mmap
					security_ops->file_mmap
				generic_file_mmap
			generic_file_vm_ops
				vma_merge
				vma_link  ▻
				file
					file->f_op->mmap
				make_pages_present
					find_vma
					get_user_pages - zero copy
						__gup_longterm_locked
				sys_remap_file_pages
					find_vma

		create_init_stack_vma
			insert_vm_struct
				vma_link

		maps_open
			proc_pid_maps_op
				m_start
					get_task_mm
					get_gate_vma
					find_vma  ▻

===== uevent

			uevent_sock_list uevent_sock
			kobject_uevent_init	lkm2
				return register_pernet_subsys(&uevent_net_ops);
					uevent_net_ops
						uevent_net_init
							NETLINK_KOBJECT_UEVENT
							uevent_sock_list
							netlink_kernel_create(net, NETLINK_KOBJECT_UEVENT,

			kobject_uevent	lkm2 120 calls
				uevent_ops
				kobject_uevent_env
					uevent_ops->filter
					uevent_sock_list
						netlink_broadcast_filtered
							do_one_broadcast
								netlink_broadcast_deliver
					call_usermodehelper
						call_usermodehelper_keys

		__netlink_create
			netlink_ops
				netlink_sendmsg
					netlink_broadcast
						netlink_broadcast_filtered

	ftrace=function
	trace_buf_size=50M

	? ftrace_filter=usb*
	? ftrace_notrace=wait_for_xmit,pfn_valid,setup_per_zone_wmarks

	echo 1 > /proc/sys/kernel/ftrace_dump_on_oops

	mcount


===== configfs
		API
			CONFIGFS_ATTR_STRUCT
				configfs_attribute
					CONFIGFS_ATTR
				show
				store
			config_item_init_type_name
				config_item_init
			config_group_init
				config_item_init
					kref_init
						atomic_set
					INIT_LIST_HEAD
			configfs_subsystem
				configfs_register_subsystem
					configfs_attach_group
				configfs_unregister_subsystem
					configfs_detach_group
				config_group
					config_item
					configfs_detach_group
					configfs_detach_group
						ci_namebuf  ▻ dirname
						ci_name
						config_item_type ci_type
							ct_item_ops: configfs_item_operations
								.show_attribute
								.store_attribute
							.ct_attrs
								configfs_attribute *
									.ca_name  ▻ filename
									.ca_mode

		configfs_example_init
			mkdir -p config; mount -t configfs none config; modprobe configfs_sample
			cat config/01-childless/storeme
			config_group_init
			childless <- to_childless <- config_item
				configfs_subsystem
			x childless_attribute
			example_subsys : configfs_subsystem
				childless_subsys : childless
					config_item_type
					childless_type
						childless_attrs
							childless_attr_showme
							childless_attr_storeme
								childless_storeme_show
									to_childless
										config_item
										to_configfs_subsystem
											to_config_group
								childless_storeme_store
						L3
							x childless_item_ops
								x childless_attr_show
								x childless_attr_store
				simple_children_subsys, simple_children_type
					simple_children_item_ops
						simple_children_release
							to_simple_children
								simple_children
									config_group
								container_of
					simple_children_group_ops :configfs_group_operations, simple_children_make_item
						.make_item
							simple_children_make_item
								kzalloc simple_child
						config_item_init_type_name
							config_item_init
						simple_child_type
							simple_child_item_ops
								simple_child_release
							simple_child_attrs
								CONFIGFS_ATTR
								simple_child
									config_item
								simple_child_attr_storeme
									simple_child_storeme_store
									simple_child_storeme_show
										to_simple_child
											container_of
									config_item
										config_item_init_type_name  ▻
				group_children_subsys
					group_children_type
						group_children_group_ops
							group_children_make_group
								config_group_init_type_name
									config_item_set_name
									config_group_init  ▻
						group_children_attrs
							group_children_attr_description
								group_children_description_show
			mutex_init
			configfs_register_subsystem  ▻
			x CHILDLESS_ATTR
			x CHILDLESS_ATTR_RO

		dynamic_netconsole_init

		configfs_file_operations
			configfs_read_file
			fill_read_buffer
				show_attribute
	misc_init
		misc_fops
			misc_open
				misc_mtx

	debugfs
		debugfs_create_file

	dma
		dma_ops
		request_dma
		enable_dma
		disable_dma

	struct scatterlist
		sg_init_one
			sg_init_table
			sg_set_buf
			sg_set_page
				sg_assign_page

	platform_driver	ldt
		device_driver driver
		probe
			platform_set_drvdata
				dev_set_drvdata
					device_private_init
					driver_data
			platform_get_drvdata
				dev_get_drvdata
					dev->driver_data

		<-
		module_platform_driver
			platform_driver_register  ▻

	dev_get_platdata
		dev->platform_data

	platform_device
		name
		id
		..
		platform_device_register

	->
		of_device_alloc
			platform_device_alloc

	platform_device_register_simple
		platform_device_register_resndata
			platform_device_info
			platform_device_register_full
				platform_device_alloc
				ERR_PTR

	data structures
		kfifo	ldt
		__kfifo
			DECLARE_KFIFO
		kfifo_in_spinlocked
			spin_lock_irqsave  ▻
				kfifo_in
			spin_unlock_irqrestore  ▻
		idr
			DEFINE_IDR
			idr_init
		list_head  ▻
		rb_root
			RB_ROOT
			rb_insert_color

	platform_driver
		platform_driver_probe	register driver for non-hotpluggable
			platform_driver_register
				driver_register  ▻
					bus_add_driver
						driver_attach

	arch_initcall
		__define_initcall
	__initcall
		device_initcall
			__define_initcall

	module_init
		!MODULE
			__initcall  ▻

	module_param lkm2
		module_param_named
			module_param_call
				__module_param_call
					kernel_param  ▻
	module_exit

	pm_init
	kernel/sys.c
		sys_reboot
			stop_this_cpu
				set_cpu_online

			kernel_restart
				kernel_restart_prepare
					device_shutdown
			machine_restart
				arch_reset
					cpu_reset
			kernel_halt
			kernel_kexec
			hibernate

==== bitops
		#define flags_test(flags, mask) (((flags) & (mask)) == (mask))

		context:
			GENMASK

		TEST_OPCODE

		Neighbor functions:
			set_mask_bits  test_bit
		regmap_update_bits_check

		similar func:
			regmap_set_bits
			regmap_test_bits https://lwn.net/Articles/821711/
			TEST_FLAGS drivers/staging/rtl8723bs/include/basic_types.h
			CHK_FLAGS drivers/media/pci/solo6x10/solo6x10.h
			HAS_CAP drivers/soc/mediatek/mtk-pmic-wrap.c
			MTK_HAS_CAPS drivers/net/ethernet/mediatek/mtk_eth_soc.h
			MTK_HAS_FLAGS
			any_allowed
			TEST_OPCODE
			BITS_SET
				drivers/net/ethernet/qualcomm/emac/emac-mac.h
				drivers/gpu/drm/i915/display/intel_display_power.c
			all_bits_set
			TESTHI
		TEST_FLAG


: {{The Linux Kernel/id|regmap_update_bits}}
: {{The Linux Kernel/id|regmap_write}}
: {{The Linux Kernel/id|regmap_read}}
: {{The Linux Kernel/id|regmap_reg_range}}
: {{The Linux Kernel/id|regmap_bulk_read}}
: {{The Linux Kernel/id|devm_regmap_init_i2c}}
: {{The Linux Kernel/id|regmap_set_bits}}
: {{The Linux Kernel/id|regmap_field_write}}
: {{The Linux Kernel/id|regmap_bulk_write}}
: {{The Linux Kernel/id|regmap_clear_bits}}
: {{The Linux Kernel/id|regmap_write_bits}}
: {{The Linux Kernel/id|regmap_config}}
: {{The Linux Kernel/id|struct clk_regmap_gate_data}}
: {{The Linux Kernel/id|struct regmap_range}}
: {{The Linux Kernel/id|regmap_read_poll_timeout}}
: {{The Linux Kernel/id|devm_regmap_init_mmio}}


		# 3K cases in 1.5K files:
		grep -r --include '*.[ch]' -Pzo ".* & (.*)\) [!=]=[ \n].*\1.*\n"

		# Complex cases, 275 results:
		grep -r --include '*.[ch]' -Pzo ".* & (\(.*\))\) [!=]=[ \n].*\1.*\n"

		example in clone3_args_valid

		vim -c '/& \((.*)\)) [!=]=[ \n].*\1'

		vim -c ':%s/(\(\w\+\) & (\(.*\))) ==[ \n]\s*(\2)/flags_test(\1, \2)/g | %s/(\(\w\+\) & (\(.*\))) !=[ \n]\s*(\2))/!flags_test(\1, \2)/g' \
			`grep -l -r --include '*.[ch]' -Pzo ".* & (\(.*\))\) [!=]=[ \n].*\1.*\n" kernel`

		grep -l -r --include '*.[ch]' -Pzo ".* & (\(.*\))\) [!=]=[ \n].*\1.*\n" \
		| xargs -n 1 \
		vim -e -c '%s/(\([^(]\+\((\w\+)\)\?\) & (\([^)]*\))) ==[ \n]\s*(\3)/flags_test(\1, \3)/ge' \
			-c '%s/(\([^(]\+\) & (\([^)]*\))) !=[ \n]\s*(\2)/!flags_test(\1, \2)/ge' \
			-c 'wq'

		grep -r --include '*.[ch]' -Hn ' & \(.*\)) == \1\>'

	ioctl
		_IOC_DIR
		_IOC_DIR _IOC_TYPE  _IOC_NR _IOC_SIZE
		_IO _IOC

==== DM

Linux Driver Model

		include/linux/device.h
				struct device
					▻ struct class
					dev_name
						dev->init_name
				struct device_link

				device_driver
					dev_pm_ops *pm
					-> bus_type
					-> module
					driver_register
					bus_type
						bus_register	drivers/base/bus.c lkm2
						subsystem
						bus_attribute
							attribute
							device_attribute
						driver_attribute
					kobject

					probe

				buses_init
					kset_create_and_add  ▻
				classes_init
					kset_create_and_add  ▻
					class_kset

					/sys/class/ lkm2
					...

					class_register
						__class_register
							kset_register  ▻
					class_create
						__class_create
							__class_register  ▻
				devices_init
					kset_create_and_add  ▻
					kobject_create_and_add  ▻
					subsystem_register  ▻

		cdev_add
		dev_set_name
		device_register
		-------------------------
		xxx
		!		device_create device_create_vargs

					device_register
						device_initialize
							kobject_init
						device_add
							device_add_attrs drivers/base/core.c
								3.2.54 device_add_attributes(dev, class->dev_attrs);
									device_create_file
								device_add_groups(dev, class->dev_groups);
									sysfs_create_groups
										sysfs_create_group
											internal_create_group
												kernfs_create_dir
													create_files
														sysfs_add_file_mode_ns
															sysfs_file_kfops_rw
																sysfs_kf_seq_show
																	ops->show
							bus_probe_device dev
								device_attach device dev
									bus_for_each_drv
									__device_attach
										struct device *dev = data
										driver_match_device  ▻
										driver_probe_device
											really_probe
												driver_sysfs_add
												probe
							device_create_file
								sysfs_create_file  ▻
				device_driver  ▻
			struct class_device
					class_device_create
						class_device_register
							class_device_initialize
							class_device_add
								class_device_create_file
									sysfs_create_file  ▻
								class_device_add_groups
									sysfs_create_group
					nouveau_hwmon_show_name
				struct class
					subsystem include/linux/kobject.h
					class_create  ▻
				kobject

		linux/kobject.h
			kobject
				kref
				??  hot_plug
				kobject_add -- sysfs
					kobject_add_varg
						kobject_set_name_vargs
						kobject_add_internal
							create_dir
								sysfs_create_dir
									create_dir  ▻
								populate_dir
								sysfs_create_file  ▻
			kset
				kset_create_and_add
					kset_uevent_ops *uevent_ops,
					kset_create
						kobject_set_name
							kobject_set_name_vargs
					kset_register
						kset_init
						kobject_add_internal  ▻
						kobject_uevent  ▻
			kobj_type
			xxx subsystem - sysfs
				subsystem_register
					x subsystem_init
					kset_register  ▻
				devices_subsys

			Documentation/kobject.txt

		/dev
			sys_mknod
				sys_mknodat
					new_decode_dev
					vfs_mknod
						security_inode_mknod

						ext3_mknod
							init_special_inode
								def_blk_fops
									blkdev_open
								def_chr_fops
									chrdev_open
										kobj = kobj_lookup(cdev_map, inode->i_rdev, &idx);
										inode->i_cdev cdev
										filp->f_op->open
								def_fifo_fops
									fifo_open

		inode	near i_hash

			union {
				struct pipe_inode_info	*i_pipe;
				struct block_device	*i_bdev;
				struct cdev		*i_cdev;
			};
			inode_operations
				setattr
		block_device

		linux/cdev.h
		fs/char_dev.c
		cdev
			chrdev_init
				cdev_map lkm2
					kobj_map_init(base_probe, &chrdevs_lock);
					request_module
						call_usermodehelper  ▻
			def_chr_fops
			cdev_alloc
			cdev_init
				kobject_init
			cdev_add  ▻
			register_chrdev - old one
				file_operations
				__register_chrdev
					__register_chrdev_region  ▻
					cdev_add lkm2
						kobj_map cdev_map
			chrdev_open
			register_chrdev_region
				__register_chrdev_region
					chrdevs
		vfsmount lkm3
			dentry_open

	system files
		sysfs  ▻

==== boot

	starts
		efi32_stub_entry arch/x86/boot/compressed/efi_mixed.S
		efi32_pe_entry

	arch/x86/boot/compressed/vmlinux.lds.S
		@startup_64
		@startup_32

	arch/x86/boot/compressed/head_64.S
		startup_64

	arch/x86/kernel/vmlinux.lds.S
		@startup_64
		@startup_32
	arch/x86/kernel/head_64.S
		startup_64
			initial_code
				secondary_startup_64_no_verify
			__startup_64
	arch/x86/kernel/head_32.S
		startup_32


	arch/x86/boot/compressed/efi_mixed.S
		efi32_pe_entry
			efi32_entry
		efi32_stub_entry if CONFIG_EFI_HANDOVER_PROTOCOL
			efi32_entry
				startup_32 arch/x86/kernel/head_32.S
					startup_64_mixed_mode arch/x86/boot/compressed/efi_mixed.S
						efi_stub_entry ->
							efi_pe_entry drivers/firmware/efi/libstub/x86-stub.c
					startup_64 arch/x86/boot/compressed/head_64.S
						


*                                                             +--------------+
*  +------------------+     +------------+            +------>| efi_pe_entry |
*  | efi32_pe_entry   |---->|            |            |       +-----------+--+
*  +------------------+     |            |     +------+----------------+  |
*                           | startup_32 |---->| startup_64_mixed_mode |  |
*  +------------------+     |            |     +------+----------------+  |
*  | efi32_stub_entry |---->|            |            |                   |
*  +------------------+     +------------+            |                   |
*                                                     V                   |
*                           +------------+     +----------------+         |
*                           | startup_64 |<----| efi_stub_entry |<--------+
*                           +------------+     +----------------+

	EFI

	drivers/firmware/efi/libstub/efistub.h

	arch/arm/boot/compressed/efi-header.S
		optional_header
			efi_pe_entry

	common_startup_64 arch/x86/kernel/head_64.S
		secondary_startup_64
			initial_code = x86_64_start_kernel
			x86_64_start_kernel ->

		initial_code -> start_secondary

drivers/firmware/efi/libstub/x86-stub.c
	efi_pe_entry
	efi64_stub_entry arch/x86/boot/compressed/efi_mixed.S
	efi_handover_entry
		efi_stub_entry drivers/firmware/efi/libstub/x85-stub.c
			efi_decompress_kernel
				efi_info
					efi_printk
						efi_puts
							efi_char16_puts
								efi_system_table
								efi_call_proto
			efi_load_initrd
			efi_retrieve_eventlog
			setup_graphics
				setup_uga
			setup_efi_pci
				EFI_PCI_IO_PROTOCOL_GUID
				efi_bs_call
				handle_protocol
			setup_unaccepted_memory
			sev_enable AMD SEV (Secure Encrypted Virtualization)
				CONFIG_AMD_MEM_ENCRYPT ?
			efi_5level_switch
				native_load_gdt
				la57_toggle
					trampoline_32bit_src
			exit_boot
				allocate_e820 E820 is the BIOS-provided memory map
					efi_get_memory_map
					alloc_e820ext
					efi_bs_call
				@exit_boot_func
					efi_loader_signature
					efi_set_u64_split
				efi_exit_boot_services
					efi_get_memory_map
					efi_bs_call
				setup_e820
					add_e820ext
			enter_kernel
				kernel_addr

	do_boot_cpu



	-> secondary_startup_64_no_verify
		start_secondary
			cpu_startup_entry
				do_idle ->
					cpuidle_idle_call
						cpuidle_enter
							cpuidle_enter_state
							poll_idle
								asm_sysvec_apic_timer_interrupt ->
			set_cpu_online



	check IDT
		idt_table
		store_idt


init/main.c


	kernel_init
		kernel_init_freeable
			cache_aps_init
				calls stop_machine with cache_rendezvous_handler and cpu_online_mask.
				stop_machine kernel/stop_machine.c
					cpus_read_lock
					stop_machine_cpuslocked
						.num_threads = num_online_cpus
						.active_cpus = cpus,
						cpu_online_mask
						&multi_cpu_stop
						set_state(&msdata, MULTI_STOP_PREPARE);
						stop_cpus
							mutex_lock(&stop_cpus_mutex);
							__stop_cpus
								cpu_stop_init_done
								queue_stop_cpus_work
									&per_cpu(cpu_stopper.stop_work, cpu)
									cpu_stopper
									for_each_cpu(cpu, cpumask) {
									cpu_stop_queue_work
										__cpu_stop_queue_work
											wake_q_add ->
										wake_up_q
						...
							cpu_stopper_thread ->
								multi_cpu_stop ->

	cpu_stop_init
		for_each_possible_cpu(cpu) {
		cpu_stop_threads
			&cpu_stopper_thread
		smpboot_register_percpu_thread



		head.S
			startup arch/sh/boot/compressed/head.S

				decompress_kernel_addr:
				decompress_kernel
					gunzip lib/inflate.c
						output_ptr = (unsigned long)&_text+0x20001000; _start
						inflate
							gzip_mark
							flush_output
								flush_window misc.c

				kernel_start_addr = _text + 0x1000

			build/arch/sh/kernel/head.S:75
			./arch/arm/mach-lpc22xx/head.S
				_stext build/vmlinux 88002000
				start_kernel  ▻ arch/sh/kernel/head.S :75

	secondary_startup_64_no_verify
	- x86_64_start_kernel
		cr4_init_shadow
			https://en.wikipedia.org/wiki/Control_register#CR4
		idt_setup_early_handler
			idt_descr
				idt_table
			early_idt_handler_array
			set_intr_gate
			load_idt
				native_load_idt
	-- x86_64_start_reservations
		x86_early_init_platform_quirks
		start_kernel init/main.c
			boot_cpu_init
				set_cpu_active
					cpumask_set_cpu
						cpumask_check
					__cpu_active_mask cpu_active_mask
				set_cpu_present
					cpumask_set_cpu
					cpumask_clear_cpu
					__cpu_present_mask cpu_present_mask
			setup_arch	lkm2
				init_hypervisor_platform
				tsc_early_init
					tsc_enable_sched_clock
						get_loops_per_jiffy
				arch/x86/kernel/setup.c
				sh_mv_setup
				enable_early_printk
					scif_sercon_init
						SCIF_REG	0xfffe9800
					register_console
						console_drivers
				platform_setup
				vmi_init
				early_trap_init  ▻
				early_cpu_init
				early_ioremap_init
				paging_init  ▻
				num_physpages lkm2
				get_num_physpages
				initmem_init
					e820_register_active_regions
						e820_find_active_region
						add_active_range
					num_physpages = max_low_pfn

				x86_init.oem.arch_setup
				setup_memory_map
					x86_init.resources.memory_setup
						default_machine_specific_memory_setup  ▻
				arm_memblock_init
					initrd_start = __phys_to_virt(phys_initrd_start);
				wallclock_init
					x86_wallclock_init
						x86_platform.get_wallclock = get_rtc_noop;
						x86_platform.set_wallclock = set_rtc_noop;
				topology_init_possible_cpus
					set_cpu_possible
						__cpu_possible_mask

			page_alloc_init
			parse_early_param
				boot_command_line
				parse_early_options
			trap_init near early_trap_init arch/x86/kernel/traps.c
				cpu_init
					syscall_init
						entry_SYSCALL_64 arch/x86/entry/entry_64.S
							do_syscall_64
								sys_call_table
							swapgs_restore_regs_and_return_to_usermode
								native_iret
									native_irq_return_iret
										iretq
						USERGS_SYSRET64
							swapgs
							sysretq
				see also early_trap_init
				set_system_trap_gate
				x set_system_gate
				lcall7/lcall27 call gates; @unix (./arch/i386/kernel/entry.S)

				set_system_trap_gate(SYSCALL_VECTOR, &system_call);
				SYSCALL_VECTOR 0x80

			mm_init	near thread_info_cache_init init/main.c lkm2
				page_cgroup_init_flatmem
				mem_init	lkm2
					memblock_free_all
						totalram_pages_add
							_totalram_pages
					pci_iommu_alloc
					pfn_to_page
						pfn = Page Frame Number
						...
					num_physpages
					nr_free_pages  ▻
					totalhigh_pages
				kmem_cache_init	lkm2	slob slab slub
					SLOB	EMBEDDED Simple Allocator
					SLAB	regular slab allocator
					SLUB	Unqueued Allocator
				pgtable_cache_init
				vmalloc_init	lkm2
					vmlist	lkm2
			vfs_caches_init
				mnt_init
					init_rootfs

			sched_init  ▻
			parse_options
			console_init
			rest_init  ->

	x kernel_thread_helper > kernel_init > prepare_namespace > mount_root > mount_block_root

	cache_ap_register
		cpuhp_setup_state_nocalls
		CPUHP_AP_CACHECTRL_STARTING,
		cache_ap_online
		cache_ap_offline


===== kernel_init

			kernel_init
				kernel_init_freeable

				init in kernel_thread, near run_init_process
					populate_rootfs via rootfs_initcall
						initrd_start
						__initramfs_start
						unpack_to_rootfs
							write_buffer
								do_start
									header_buf
									read_into

							gunzip
					do_basic_setup
						driver_init
							devices_subsys
							devices_init  ▻
						sysctl_init (proc)  ▻
						pci_init
						x	pcibios_init
						x		pcibios_resource_survey
						x			pcibios_allocate_bus_resources
						sock_init
							sk_init
						start_context_thread
							context_thread
								schedule
						do_initcalls
							populate_rootfs  ▻ via rootfs_initcall

					prepare_namespace # mounts
						saved_root_name
							< __setup("root=", root_dev_setup);
						initrd_load init/do_mounts_initrd.c if CONFIG_BLK_DEV_INITRD
							rd_load_image if CONFIG_BLK_DEV_RAM
								identify_ramdisk_image
									SQUASHFS_MAGIC          0x73717368
									"filesystem found at block"
								"RAMDISK: image too big"

						mount_root
							create_dev
								sys_mknod
							mount_block_root
								root_mount_data
								get_fs_names
									get_filesystem_list
								do_mount_root
									sys_mount  ▻
						rd_load_disk
							rd_load_image  ▻
					run_init_process
						kernel_execve
						old execve
							sys_execve  ▻

						/sbin/init
							init_main (busybox)
								init_action_list
									parse_inittab
								console_init
								/etc/inittab

		init_cramfs_fs
			cramfs_uncompress_init
				register_filesystem
				cramfs_fs_type
					cramfs_get_sb
						cramfs_fill_super
							cramfs_uncompress_block

===== initrd

	paging_init
		pagetable_init

	phys_initrd_start

	CONFIG_BLK_DEV_INITRD

	arch/arm/kernel/setup.c

	early_initrd  initrd=start,size

}

==== Virtualization
{	virtualization
	kvm
		Address: virtual/physical/frame guest/host
			gva_t guest virtual address
			gpa_t guest physical address
			gfn_t guest frame number
			hva_t host virtual address
			hpa_t host physical address
			hfn_t host frame number
		other
			tdp_page_fault
		PMU Performance Monitoring Unit
	kvm_guest_init arch/x86/kernel/kvm.c
		kvm_para_has_feature
		kvm_para_has_hint
	pci_sriov_get_totalvfs
	pci_num_vf
	svm_init
		kvm_init
	vmx_init
		kvm_init
	kvm_init
		kvm_arch_init
		kvm_chardev_ops
		kvm_dev_ioctl ./virt/kvm/kvm_main.c
			kvm_dev_ioctl_create_vm
				kvm
					kvm_vm_fops
						kvm_vm_ioctl,
							kvm_vm_ioctl_create_vcpu
								kvm_arch_vcpu_create
									kvm_x86_ops-
									vmx_create_vcpu
									svm_create_vcpu
				kvm_create_vm

		vmx_x86_ops
			hardware_setup
			vmx_set_ms
				MSR (Model Specific Register)
				kvm_arch_dev_ioctl
					KVM_GET_MSRS
		kvm_setup_async_pf
		Tsirkin
			kvm_vm_ioctl_check_extension
			vhost_new_msg
			virtcons_remove
			remove_vqs virtio_device_for_each_vq
	drivers/virtio
		SRIOV Single Root I/O Virtualization (SRIOV SR-IOV)
			numvfs sriov_numvfs_show
				num_VFs
		virtio_pci_driver
			virtio_pci_probe
				virtio_pci_modern_probe
			virtio_pci_sriov_configure
				pci_enable_sriov ->
		virtio_pci_modern_probe
		PCI_EXT_CAP_ID_SRIOV
		PCI_EXT_CAP_ID_MRIOV
	pci_init_capabilities
		pci_msi_setup_pci_dev
			pci_find_capability
				__pci_bus_find_cap_start
					PCI_STATUS
					pci_bus_read_config_word
				pci_find_next_capability
			PCI_CAP_ID_MSI
			PCI_CAP_ID_MSIX
			msix_cap
			pci_msix_clear_and_set_ctrl
		trl_();
		trvs_(dev_name(dev));
		trvd(irq);
		pci_iov_init
			pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
				pci_find_next_ext_capability
					pci_read_config_dword
					PCI_EXT_CAP_ID
					PCI_EXT_CAP_NEXT
			sriov_init
				pci_sriov
				PCI_SRIOV_CTRL
				PCI_SRIOV_TOTAL_VF
				is_physfn

		pci_enable_sriov
			is_physfn
			CONFIG_PCI_IOV
				pci_sriov_configure_simple
					sriov_enable
				(pci_disable_sriov)
			callers
				be_sriov_enable
				lpfc_sli_probe_sriov_nr_virtfn
				ixgbe_enable_sriov
			sriov_enable
				PCI_SRIOV_INITIAL_VF
				PCI_SRIOV_CAP
				PCI_SRIOV_CAP_VFM

}

==== NMI watchdog

	include/linux/nmi.h
	test on 5.14.0-284.69.1.rt14.354.el9_2.x86_64
		https://access.redhat.com/downloads/content/kernel-rt-devel/5.14.0-284.69.1.rt14.354.el9_2/x86_64/fd431d51/package
	kernel/watchdog.c
		lockup_detector_init
			lockup_detector_setup
				__lockup_detector_reconfigure
					lockup_detector_update_enable
						NMI_WATCHDOG_ENABLED
					softlockup_start_all
						watchdog_cpumask
		watchdog_enable
			hrtimer_init
			hrtimer_start
			watchdog_timer_fn
				watchdog_interrupt_count
					hrtimer_interrupts
				6 watchdog_hardlockup_kick
			6 watchdog_hardlockup_enable
				6 watchdog_hardlockup_touch_cpu
				watchdog_next_cpu
					watchdog_cpus
			watchdog_nmi_enable
				hardlockup_detector_perf_enable
					hardlockup_detector_event_create
						perf_event_create_kernel_counter
						watchdog_overflow_callback
							is_hardlockup
		CONFIG_SOFTLOCKUP_DETECTOR
		CONFIG_HARDLOCKUP_DETECTOR
		touch_nmi_watchdog
			arch_touch_nmi_watchdog
			touch_softlockup_watchdog

		lockup_detector_check
			watchdog_sysctl_init
				watchdog_sysctls
					proc_watchdog_cpumask
				CONFIG_LOCKUP_DETECTOR
				nmi_watchdog
					echo 1 > /proc/sys/kernel/nmi_watchdog

== Multitasking

	can_migrate_task
		kthread_is_per_cpu
			KTHREAD_IS_PER_CPU


<<_real_time_preemption>>

{

=== kthreads


Affining kthreads follow either of four existing different patterns:

1) Per-CPU kthreads must stay affine to a single CPU and never execute
   relevant code on any other CPU. This is currently handled by smpboot
   code which takes care of CPU-hotplug operations.
	kthread_create_on_cpu
	<
   	smpboot_register_percpu_thread hotplug_threads smpboot_park_thread
   <
   	wti_init idle_inject_init cpuhp_threads_init irq_work_init_threads
   	rcu_spawn_core_kthreads spawn_ksoftirqd> cpu_stop_init> net_dev_init


2) Kthreads that _have_ to be affine to a specific set of CPUs and can't
   run anywhere else. The affinity is set through kthread_bind_mask()
   and the subsystem takes care by itself to handle CPU-hotplug operations.

	kthread_bind_mask
	< do_map_benchmark sugov_kthread_create create_worker init_rescuer


3) Kthreads that prefer to be affine to a specific NUMA node. That
   preferred affinity is applied by default when an actual node ID is
   passed on kthread creation, provided the kthread is not per-CPU and
   no call to kthread_bind_mask() has been issued before the first
   wake-up.

	kthread_affine_node

	kthread_create_on_cpu
	< 
   	suspend_tests bnx2fc_cpu_online bnx2i_cpu_online qedi_cpu_online
   	__smpboot_create_thread pktgen_create_thread page_frag_test_init

	kthread_create_on_node
	<
   	mtip_block_initialize svc_get_sh_memory __cpu_map_entry_alloc do_map_benchmark
	create_worker kcompactd_run kswapd_run svc_start_kthreads


4) Similar to the previous point but kthreads have a preferred affinity
   different than a node. It is set manually like any other task and
   CPU-hotplug is supposed to be handled by the relevant subsystem so
   that the task is properly reaffined whenever a given CPU from the
   preferred affinity comes up. Also care must be taken so that the
   preferred affinity doesn't cross housekeeping cpumask boundaries.

   kthread_affine_preferred




	devtmpfs_init	
		kthread_run(devtmpfsd, &err, "kdevtmpfs");
	oom_init
		oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper");
			&oom_reaper
		

	kthreads_init
		cpuhp_setup_state
		&kthreads_online_cpu 
			<- cpuhp_invoke_callback
			kthreads_hotplug_lock
			kthreads_hotplug
			!kthread_is_per_cpu
			kthread_fetch_affinity
				preferred_affinity - only rcu_thread_affine_rnp
				cpumask_of_node
					node_to_cpumask_map
				cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_KTHREAD));
			set_cpus_allowed_ptr ->

	kthread_affine_preferred
		kthreads_hotplug_lock
		kthreads_hotplug
		preferred_affinity
		kthread_fetch_affinity ->


	kthreadd
		create_kthread
			kernel_thread
			@kthread
				ass pointer
				set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_KTHREAD));
				kthread_affine_node
					!kthread_is_per_cpu
					housekeeping_affine
						housekeeping
						set_cpus_allowed_ptr
					kthreads_hotplug_lock
					kthreads_hotplug
					kthread_fetch_affinity ->

	kthread_run_on_cpu
		<
			test_ringbuffer
			kallsyms_test_init
		kthread_create_on_cpu
			cpu_to_node(cpu)
			kthread_create_on_node ->
			kthread_bind ->
		kthread_bind
		wake_up_process


	kthread_create_worker_on_cpu < erofs_init_percpu_worker
		kthread_bind
			__kthread_bind ->
				__kthread_bind_mask ->

	kthread
		kthread_run	ldt
			kthread_create  ▻
		kthread_should_stop
			test_bit
			KTHREAD_SHOULD_STOP
		kthread_stop
			to_kthread
				PF_KTHREAD
				worker_private
		kthread_delayed_work
			DEFINE_KTHREAD_DELAYED_WORK
				KTHREAD_DELAYED_WORK_INIT
					kthread_delayed_work_timer_fn
						kthread_insert_work
					KTHREAD_WORK_INIT

		spi_init_queue
			kthread_run_worker
			&spi_pump_messages
			kthread_init_work
		spi_start_queue
			kthread_queue_work
		spi_destroy_queue
			kthread_destroy_worker

			kthread_destroy_worker
				kthread_flush_worker
					kthread_queue_work
					wait_for_completion
		pwq_release_worker
			init_pwq
				kthread_init_work
			wq_cpu_intensive_thresh_init
				"pool_workqueue_release"
				kthread_run_worker ->
			put_pwq
				kthread_queue_work

		watchdog_kworker
			watchdog_dev_init
				"watchdogd"
				kthread_run_worker
			watchdog_timer_expired
				kthread_queue_work
			watchdog_dev_exit
				kthread_destroy_worker

		kthread_queue_work
			kthread_work
			kthread_worker
			work_list
			kthread_insert_work

		kthread_work
			kthread_init_delayed_work
				timer_setup
				kthread_init_work
			DEFINE_KTHREAD_WORK
				KTHREAD_WORK_INIT
		kthread_worker
			__kthread_init_worker
			kthread_run_worker
				kthread_create_worker
					NUMA_NO_NODE
					kthread_create_worker_on_node
						__kthread_create_worker_on_node
							&kthread_worker_fn
								trace_sched_kthread_work_execute_start
								work->func(work);
								trace_sched_kthread_work_execute_end
							__kthread_create_on_node ->

		kthread_park
			KTHREAD_SHOULD_PARK
			TASK_PARKED
			wait_task_inactive

		kthread_unpark
			KTHREAD_IS_PER_CPU
			__kthread_bind
				cpumask_of
					get_cpu_mask
						cpu_bit_bitmap
						to_cpumask
				__kthread_bind_mask ->

	kthread_bind_mask
		TASK_UNINTERRUPTIBLE
		__kthread_bind_mask
			wait_task_inactive
			raw_spin_lock_irqsave
			do_set_cpus_allowed	consider using set_cpus_allowed_ptr() instead
				affinity_context
				.user_mask = NULL
				__do_set_cpus_allowed ->
			PF_NO_SETAFFINITY


=== sched


	resched_curr	

	cond_resched

		#include <linux/sched.h>
		jiffies

	sched_load_balance
	isolcpus=domain
		HK_FLAG_DOMAIN HK_TYPE_DOMAIN ->

	Requirements to control kernel isolation/nohz_full at runtime
		schedule_user not used
			exception_enter
			schedule
			exception_exit
		preempt_schedule_notrace
	Start using bpftrace/eBPF for latency and load tests.
	frederic
		housekeeping_cpumask_set
		housekeeping_cpumask_clear
			housekeeping_cpumask_update
				rcu_nocb_cpumask_update
					cpumask_or
					cpumask_andnot


	Scheduler

		lightweight_kernel_threads

		11. Process Scheduling
		      Scheduling Policy
		      The Scheduling Algorithm
		      System Calls Related to Scheduling

		linux/sched.h
		schedule kernel/sched.c
			__schedule_loop kernel/sched/core.c
			__schedule 130 LOC
				pre_schedule
				signal_pending_state
				trace_sched_switch
					"sched_switch"
					TP_printk
				context_switch
					switch_mm mm_struct
					switch_to
						__switch_to
					finish_task_switch
				x post_schedule

		sys_nice
			set_user_nice
				NICE_TO_PRIO

		rq runqueue s
			runqueue_t

		sched_init
			hrtick_rq_init
				hrtick_csd
				__hrtick_start
					__hrtick_restart
				hrtick_timer.function = hrtick
					task_tick ->
				hrtick
					update_rq_clock
					sched_class->task_tick
						task_tick_rt
							requeue_task_rt
			init_idle ->

		setup_IO_APIC Advanced Programmable Interrupt Controller
			setup_IO_APIC_irqs
				...
					set_intr_gate(vector, interrupt[irq]); ->
						...
							pack_gate

		init_IRQ	/arch/x86/kernel/irqinit.c
			x86_init.irqs.intr_init
		native_init_IRQ lkm2
			idt_setup_apic_and_irq_gates
				set_intr_gate	lkm2
				set_intr_gate(vector, interrupt[irq]);
					init_idt_data
					idt_table
					idt_setup_from_table ->
						idt_init_desc ->
						write_idt_entry ->
		interrupt source/arch/i386/kernel/entry.S
		irq_entries_start (arch/i386/kernel/entry.S)
			common_interrupt:
				do_IRQ  ▻
				ret_from_intr
					resume_kernel
						preempt_schedule_irq
							schedule  ▻

		load_balance

		start_kernel
		rest_init
			kernel_init (via kernel_thread:)
				kernel_init_freeable
					smp_init
						idle_threads_init ->
						cpuhp_threads_init
							cpuhp_threads
							cpuhp_init_state
								cpuhp_state
							smpboot_register_percpu_thread ->
					do_basic_setup  ▻
					x ? init_post
			run_init_process  ▻
			kernel_thread(kthreadd
			kthreadd_task
			cpu_startup_entry
					do_idle ->
						tick_nohz_idle_enter
							tick_sched
							ts->inidle = 1;
							[TS_FLAG_INIDLE] = 1
							tick_nohz_start_idle
								ts->idle_entrytime = ktime_get();
								ts->idle_active = 1;
								sched_clock_idle_sleep_event
									sched_clock_cpu(smp_processor_id());
							tick_sched_flag_set(ts, TS_FLAG_INIDLE);
						!need_resched
						cpu_is_offline
							cpu_online_mask
						cpuhp_report_idle_dead
							rcutree_report_cpu_dead
							play_dead
							smp_call_function_single
							cpuhp_complete_idle_dead
								complete_ap_thread
									done_up
									done_down
						arch_cpu_idle_dead
						arch_cpu_idle_enter
						tick_nohz_idle_restart_tick
						cpu_idle_poll
							cpu_relax
						cpuidle_idle_call ->
							tick_nohz_idle_stop_tick ->
							tick_nohz_tick_stopped
								TS_FLAG_STOPPED
						arch_cpu_idle_exit
						tick_nohz_idle_exit
							ts->inidle = 0;
							tick_nohz_idle_update_tick
								tick_nohz_account_idle_time
									ts->idle_exittime = now;
			cpuidle_enter_state
				err: default_idle_call
					trace_cpu_idle
					arch_cpu_idle
						x86_idle
							default_idle
								raw_safe_halt
									arch_safe_halt
										native_safe_halt
											hlt
				enter = &intel_idle
					__intel_idle
						mwait_idle_with_hints

				trace_cpu_idle
			cpu_idle_poll


	tick_nohz_idle_exit
		tick_nohz_idle_update_tick
			tick_nohz_restart_sched_tick ^
				timer_clear_idle ->

	idle_threads_init
		idle_init
			idle_threads
			fork_idle
				init_idle_pids
				init_idle
					INIT_TASK_COMM "swapper"
					kthread_set_per_cpu
					set_cpus_allowed_common

	bringup_cpu
		idle_thread_get(cpu);
			idle_threads
		__cpu_up
			cpu_up
				= native_cpu_up
					common_cpu_up
		bringup_wait_for_ap
			cpuhp_kick_ap

	do_cpu_up
		_cpu_up

	tick_nohz_idle_stop_tick
		can_stop_idle_tick
			report_idle_softirq
				local_softirq_pending
					local_softirq_pending_ref
						softirq_pending

		__tick_nohz_idle_stop_tick
			ts->timer_expires
				tracepoint:timer:*timer_expire_entry /cpu==$1/{printf("%s %u\n",probe, cpu)}
			tick_nohz_next_event
				__get_next_timer_interrupt
					base_local->is_idle = true
					tmigr_quick_check➝asm_exc_invalid_op➝exc_invalid_op➝handle_bug➝report_bug➝tmigr_quick_check➝__warn
			tick_nohz_stop_tick

		ENTRY(cpu_arm926_do_idle)
			...
			mcr	p15, 0, r0, c7, c0, 4		@ Wait for interrupt
			pm_idle

		 linux/kernel/sched.c
		task_struct

		renice

		include/linux/wait.h
		include/linux/sched.h
		DECLARE_WAIT_QUEUE_HEAD()

	UMP:
		__set_cpus_allowed_ptr
			set_cpus_allowed_ptr

=== Locking

include/linux/cleanup.h

	DEFINE_GUARD
	guard
	scoped_guard (< do_timer_create)



	Synchronizations
		5. Kernel Synchronization
		      Kernel Control Paths
		      When Synchronization Is Not Necessary
		      Synchronization Primitives
		      Synchronizing Accesses to Kernel Data Structures
		      Examples of Race Condition Prevention

		preempt_disable -> RT -> migrate_disable

		migrate_disable_switch
			SCA_MIGRATE_DISABLE
			__do_set_cpus_allowed

		migrate_disable
			p->migration_disabled++;
			preempt_lazy_disable();
				inc_preempt_lazy_count
					add_preempt_lazy_count
						preempt_lazy_count
							thread_info
		preemptible_lazy

		migrate_enable ->
			p->migration_disabled--;
			SCA_MIGRATE_ENABLE
			__set_cpus_allowed_ptr ->
				__set_cpus_allowed_ptr_locked ->
					__do_set_cpus_allowed ->

		atomic_t
		spinlock_t
		semaphore
		wait_queue_t

		include/linux/bottom_half.h
			local_bh_disable
				SOFTIRQ_DISABLE_OFFSET
				__local_bh_disable_ip
					preempt_count_add
			local_bh_enable
				SOFTIRQ_DISABLE_OFFSET
				__local_bh_enable_ip

		local_lock
			__local_lock
				!CONFIG_PREEMPT_RT
					preempt_disable ->
					local_lock_acquire(this_cpu_ptr(lock));
				CONFIG_PREEMPT_RT
					migrate_disable ->
					spin_lock(this_cpu_ptr((__lock)))
						rt_spin_lock
							__rt_spin_lock ->
	sample
		nft_inner_restore_tun_ctx
			local_bh_disable
			local_lock_nested_bh

			local_unlock_nested_bh
			local_bh_enable();
==== Local lock

local_irq_save local_irq_restore, frequent 

		local_lock_irqsave
			__local_lock_irqsave
				!rt:
					local_irq_save
						raw_local_irq_save
							arch_local_irq_save
								arch_local_save_flags
									native_save_fl
										pushf ; pop
								arch_local_irq_disable
									native_irq_disable
										cli
						raw_irqs_disabled_flags
							arch_irqs_disabled_flags
								X86_EFLAGS_IF
					local_lock_acquire

				rt:
					__local_lock ->
						migrate_disable
						spin_lock

		...
					local_irq_disable
						raw_local_irq_disable
							arch_local_irq_disable ->
						trace_hardirqs_off
		local_unlock_irqrestore
			!rt:
			__local_unlock_irqrestore
				local_lock_release
				local_irq_restore
					raw_local_irq_restore
						raw_check_bogus_irq_restore
							warn_bogus_irq_restore
						arch_local_irq_restore
							arch_irqs_disabled_flags
							arch_local_irq_enable
								native_irq_enable
									sti
			rt:
				__local_unlock
					rt:
						spin_unlock ->
						migrate_enable ->

			local_unlock
				__local_unlock
					!rt
					local_lock_release

		irqflags.h
			irqs_disabled
				raw_irqs_disabled
					arch_irqs_disabled
						arch_local_save_flags ->
						arch_irqs_disabled_flags
		local_lock_irq
			__local_lock_irq
				!rt:
				local_irq_disable
				local_lock_acquire
				rt:
				__local_lock ->
		local_unlock_irq
			__local_unlock_irq

		atomic_t
		atomic_dec_and_test
			atomic_dec_return (generic)
				atomic_sub_return
					atomic_add_return  ▻
						xadd
							__xadd

		completion
			wait_queue_head_t   ▻
			DECLARE_COMPLETION
				COMPLETION_INITIALIZER
					__WAIT_QUEUE_HEAD_INITIALIZER
			init_completion
				init_waitqueue_head
					INIT_LIST_HEAD
						prev, next
			wait_for_completion_interruptible	ldt
			wait_for_completion
				wait_for_common
					__wait_for_common
						complete_acquire
				__add_wait_queue_tail
					list_add_tail
				schedule
				if done > 0, done--
			complete
				done++

==== Mutex


		mutex	__LINUX_MUTEX_H	include/linux/mutex.h


			atomic_t		count;
			spinlock_t		wait_lock;
			struct list_head	wait_list;

			No timeout for mutex!

			mutex_lock
				!RT  kernel/locking/mutex.c
				 might_sleep();
				__mutex_trylock_fast
					atomic_long_try_cmpxchg_acquire
						raw_atomic_long_try_cmpxchg_acquire
							raw_atomic_try_cmpxchg_acquire
				__mutex_lock_slowpath
					__mutex_lock
						__mutex_lock_common ->
				x __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
				mutex_set_owner(lock);
			mutex_lock_interruptible
				__mutex_fastpath_lock_retval
				__mutex_lock_interruptible_slowpath
					__mutex_lock_common
						RT
							kernel/locking/rtmutex_api.c
								mutex_acquire_nest
									lock_acquire_exclusive
								__rt_mutex_lock ->
								lock_acquired
									CONFIG_LOCK_STAT
									__lock_acquired
						!RT
							kernel/locking/mutex.c
							schedule_preempt_disabled
							wait_lock
							ww_mutex_set_context_fastpath
								wake_up_q
							__mutex_trylock
								__mutex_trylock_common
									atomic_long_try_cmpxchg_acquire ->
			mutex_lock_killable
				__mutex_lock_killable_slowpath
					return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
			mutex_unlock
				RT
				__rt_mutex_unlock
				!RT
				__mutex_unlock_fast
					atomic_long_try_cmpxchg_release
				__mutex_unlock_slowpath
					atomic_long_try_cmpxchg_release
					wake_q_add
						__wake_q_add
							wake_q_node
							cmpxchg_relaxed
				x __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
				__mutex_unlock_slowpath
					x __mutex_unlock_common_slowpath
						wake_up_process  ▻
				x mutex_clear_owner(lock);

			lockdep
				lockdep_map
				lock_is_held
					lock_is_held_type

==== Semaphore
		semaphore	source/include/linux/semaphore.h
			spinlock_t		lock;
			unsigned int		count;
			struct list_head	wait_list;

			wait_queue_head_t  ▻
			x init_MUTEX
			sema_init	struct semaphore *sem, int val
					init_waitqueue_head

			down_interruptible
				__down_interruptible
					__down_common(sem, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT)
			down_killable
			down_timeout
			down
				__sem_acquire
					sem->count--;
				x __down_failed
					__down
						__down_common
							___down_common
								signal_pending_state
									signal_pending
										return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
									__fatal_signal_pending
										task_struct pending.signal
								schedule_timeout

						add_wait_queue_exclusive_locked
							atomic_add_negative
							__add_wait_queue_tail
						schedule
			down_trylock
			up 	lkm2
				sem->count++;
 				Unlike mutexes, up() may be called from any context and even by tasks which have never called down().
				x __up_wakeup

				__up
					wake_q_add ->
				wake_up_q
					wake_up_process
						try_to_wake_up  ▻

		rw_semaphore
		up down * writer reader

==== spinlocks

		no wait list
		only one locker
		busy loop  ▻ it can be used in interrupt context (where sleeping is not an option)

		include/linux/spinlock.h

		spinlock_t
			raw_spinlock
			spin_lock	lkm2
				raw_spin_lock
					_raw_spin_lock
						__raw_spin_lock
							preempt_disable();
							spin_acquire ->
							LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
							???
								native_queued_spin_lock_slowpath PARAVIRT_SPINLOCKS
				----
					preempt_disable ->
					spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
						lock_acquire
							__lock_acquire
								atomic_inc
									arch_atomic_inc
									atomic_add_return  ▻
										arch_atomic_add_return
											xadd
								lockdep_print_held_locks
					_raw_spin_lock(lock);
						__raw_spin_trylock
							xchgb
			spin_lock_irq()
			spin_unlock_irq()

			spin_lock_irqsave lkm2
				RT
				spin_lock include/linux/spinlock_rt.h
					rt_spin_lock ->

				!RT
				raw_spin_lock_irqsave
					_raw_spin_lock_irqsave
						SMP __raw_spin_lock_irqsave
							local_irq_save
							preempt_disable
							spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
								lock_acquire
							LOCK_CONTENDED
							do_raw_spin_lock_flags
								arch_spin_lock_flags
									arch_spin_lock
										__ticket_spin_lock
								...

						UP	__LOCK_IRQSAVE
							local_irq_save(flags); __LOCK
				x _spin_lock_irqsave
					local_irq_save
					preempt_disable  ▻
					spin_acquire  ▻
					_raw_spin_lock_flags
						__raw_spin_lock_flags
							local_irq_save
			spin_unlock_irqrestore lkm2
				rt_spin_unlock
				raw_spin_unlock_irqrestore
					_raw_spin_unlock_irqrestore
						__raw_spin_unlock_irqrestore
			spin_unlock
				raw_spin_unlock
					_raw_spin_unlock
						__raw_spin_unlock
							spin_release(&lock->dep_map, 1, _RET_IP_);
								~ lock_release
							do_raw_spin_unlock(lock);
								arch_spin_unlock
									__ticket_spin_unlock
										incb
								~ __release
									__context__
							preempt_enable();

		include/linux/wait.h
			DECLARE_WAITQUEUE
				wait_queue_entry
				__WAITQUEUE_INITIALIZER
					default_wake_function
			add_wait_queue
			wait_queue_head_t
				__wait_queue_head
					lock
					struct list_head task_list;
				DECLARE_WAIT_QUEUE_HEAD
				interruptible_sleep_on decates, uses wait_event_interruptible
					sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
						init_waitqueue_entry(&wait, current);
							__add_wait_queue
						timeout = schedule_timeout(timeout);
				wake_up_interruptible  ▻
					__wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
						->

		add_wait_queue
			__add_wait_queue
				list_add(&new->task_list, &head->task_list);
		remove_wait_queue
			__remove_wait_queue

			wake_up_interruptible ldt
			wait_event	lkm2
				__wait_event
					prepare_to_wait  ▻
					schedule  ▻
			wait_event_interruptible	ldt
				__wait_event_interruptible
					___wait_event
						prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE);
						prepare_to_wait
							spin_lock_irqsave  ▻
							__add_wait_queue  ▻
							set_current_state
							spin_unlock_irqrestore
						signal_pending
							test_tsk_thread_flag TIF_SIGPENDING
								test_ti_thread_flag task_thread_info thread_info
									test_bit flags
					schedule ->
			wait_event_interruptible_timeout
				__wait_event_interruptible_timeout
					prepare_to_wait  ▻
					signal_pending  ▻
					schedule_timeout

			DEFINE_WAIT init_wait autoremove_wake_function
			init_waitqueue_head
					__init_waitqueue_head
						lockdep_set_class_and_name
						INIT_LIST_HEAD
			wake_up
				__wake_up
					spin_lock_irqsave  ▻
					__wake_up_common_lock
					__wake_up_common
						default_wake_function
							try_to_wake_up
								task_rq_lock
									task_rq
								x 
									activate_task -> lkm2
										__activate_task
								resched_task
					spin_unlock_irqrestore  ▻

		fork()
		_exit()
		linux/kernel/fork.c


==== topology

	include/linux/sched/topology.h
	sched_domain_topology_level
		set_sched_topology
		sd_data
			sched_domain
				sched_group

	drivers/base/arch_topology.c

	update_topology_flags_workfn
		rebuild_sched_domains ->
	
	kernel/sched/topology.c
		rebuild_sched_domains_energy
			rebuild_sched_domains ->


=== Synchronizations

=== Time
	include/linux/vtime.h
		account_softirq_enter
			SOFTIRQ_OFFSET
			irqtime_account_irq
				irqtime_account_delta
					cpustat[idx] += delta;

	"clocksource"
		clocksource_attrs
			current_clocksource_store
				clocksource_select
					__clocksource_select
						timekeeping_notify include/linux/clocksource.h
							cpus = null
							stop_machine
							change_clocksource
							tick_clock_notify
=== Timers

==== Posix Timers
		include/linux/posix-timers.h
		include/linux/posix-timers_types.h
			signal_struct
				<- task_struct
				posix_cputimers
					posix_cputimer_base

		kernel/posix-timers.c
			sys_clock_getres
				CLOCK_DISPATCH
					posix_clocks
					common_clock_getres
			sys_clock_gettime
				CLOCK_DISPATCH
			sys_clock_nanosleep
				hrtimer_nanosleep ->
			sys_clock_settime
			sys_timer_create
				do_timer_create
					k_itimer
					alloc_posix_timer
						k_itimer
						posix_timers_cache
			sys_timer_delete
			sys_timer_getoverrun
			sys_timer_gettime
			sys_timer_settime

		kernel/time/posix-timers.c
			common_timer_create

		kernel/time/posix-cpu-timers.c

			set_process_cpu_timer
				tick_dep_set_signal(tsk, TICK_DEP_BIT_POSIX_TIMER);

			run_posix_cpu_timers					
				< update_process_times
				posix_cpu_timers_work_scheduled
					__run_posix_cpu_timers
						task_work_add
						posix_cputimers_work

		clock_process
			process_cpu_timer_create
				posix_cpu_timer_create
		posix_cpu_nsleep		
		do_cpu_nanosleep
			posix_cpu_timer_create
				clock_posix_cpu
					posix_cpu_nsleep
						do_cpu_nanosleep
							posix_cpu_timer_set	
								arm_timer
									tick_dep_set_signal(p, TICK_DEP_BIT_POSIX_TIMER);
										tick_nohz_dep_set_signal

		kernel/time/itimer.c
			sys_getitimer
				itimerspec64
					struct timespec64 it_interval;
					struct timespec64 it_value;
				do_getitimer
					itimer_get_remtime
						ktime_to_timeval
							ktime_t  ▻ timeval
					get_cpu_itimer
						thread_group_cputimer
			sys_setitimer
				do_setitimer
					hrtimer_start
					set_cpu_itimer
						set_process_cpu_timer

==== hrtimer

	include/linux/hrtimer.h
		hrtimer_setup
			__hrtimer_setup
				__hrtimer_init ->
				timer->function
		hrtimer_start
			hrtimer_start_range_ns ->
		hrtimer_restart
			hrtimer_start_expires
				hrtimer_start_range_ns ->
		hrtimer_cancel
			hrtimer_try_to_cancel
				hrtimer_clock_base
				hrtimer_active
				hrtimer_callback_running
				remove_hrtimer
			hrtimer_cancel_wait_running
		hrtimer_init
			debug_init
				debug_hrtimer_init
					hrtimer_debug_descr
					debug_object_init
			__hrtimer_init
				hrtimer_clockid_to_base
				timerqueue_init
					RB_CLEAR_NODE
		hrtimer_sleeper
			hrtimer timer
			task_struct *task;
		hrtimer_active
			!HRTIMER_STATE_INACTIVE
		hrtimer_mode
		hrtimer_forward_now
			timer->base->get_time() hrtimer_bases
				ktime_get ->

			hrtimer_forward
				hrtimer_get_expires
					timer->node.expires
				ktime_sub
				hrtimer_add_expires_ns
					timer->node.expires =
					timer->_softexpires =
				hrtimer_get_expires_tv64
				hrtimer_add_expires
					ktime_add_safe
						ktime_add_unsafe
		hrtimer struct
			timerqueue_node
			_softexpires
			-> hrtimer_clock_base
		hrtimer_cpu_base
			hrtimer_clock_base

		hrtimer_set_expires
			timer->node.expires = time;
			timer->_softexpires = time;

		hrtimers_init
			HRTIMER_SOFTIRQ
			open_softirq
			hrtimer_run_softirq
		debug_activate
			trace_hrtimer_start

		hrtimer_start_range_ns
			lock_hrtimer_base(timer, &flags);
				raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
			__hrtimer_start_range_ns
				HRTIMER_MODE_PINNED
				switch_hrtimer_base
					get_target_base
						timers_migration_enabled
						get_nohz_timer_target
							bpftrace -e 'kretprobe:get_nohz_timer_target {print(retval);}'
							idle_cpu

							housekeeping_cpu(cpu, HK_TYPE_TIMER))
							housekeeping_any_cpu ->
				enqueue_hrtimer
					timerqueue_add
						rb_add_cached
			hrtimer_reprogram
				hrtimer_bases
				__hrtimer_reprogram
					tick_program_event
						clock_event_device
						clockevents_program_event
							clockevents_program_min_delta
								clockevents_increase_min_delta
			unlock_hrtimer_base(timer, &flags);
				raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);

		tick_sched_timer is a timer for NOHZ mode(check)
			tick_sched_do_timer ->
			tick_sched_handle
			!tick_stopped
			hrtimer_forward ->

		CONFIG_HIGH_RES_TIMERS
			__hrtimer_peek_ahead_timers
				hrtimer_interrupt ->
			hrtimer_switch_to_hres

		timer_list ->

		/proc/timer_list
		timer_list_show
			print_cpu
				tick_get_tick_sched
				hrtimer_bases
					hrtimer_cpu_base
						hrtimer
							-> hrtimer_clock_base
						hrtimer_clock_base
							get_time
							-> hrtimer
				print_base
					print_active_timers
						timerqueue_getnext
						timerqueue_iterate_next
						print_timer
	kernel/hrtimer.c
		sys_nanosleep  ▻

	hrtimer_wakeup

	hrtimer_run_softirq
		__hrtimer_run_queues ->

	__hrtimer_run_queues
		__run_hrtimer
			timer->function
			trace_hrtimer_expire_entry "hrtimer_expire_entry"
			->tick_setup_sched_timer
			->hrtimer_wakeup
				wake_up_process ->
					try_to_wake_up  ▻
						select_task_rq ->
						psi_ttwu_dequeue
							psi_task_change
								psi_flags_change
								psi_group_change
						set_task_cpu
						ttwu_queue
							ttwu_do_activate
								ttwu_do_wakeup ->


==== timer_list


linux/kernel/timer.c


			timer_clear_idle
				is_idle = false
				trace_timer_base_idle
				tmigr_cpu_activate
					tmigr_cpu
					__tmigr_cpu_activate
						tmigr_active_up

		TIMER_DEFERRABLE -> BASE_DEF, !BASE_STD
		init_timers
			open_softirq
			TIMER_SOFTIRQ
			run_timer_softirq lkm2
		timer_list
			DEFINE_TIMER
			timer_setup
				__init_timer
					init_timer_key
						dinit_idle_pidso_init_timer
							timer->function
			get_timer_this_cpu_base (was get_target_base) kernel/time/timer.c
				timers_migration_enabled
				TIMER_PINNED -> BASE_LOCAL
				get_timer_cpu_base
					per_cpu_ptr
					timer_bases	DEFINE_PER_CPU
						timer_base
							timer_list
								expires
				!CONFIG_SMP
				get_timer_this_cpu_base
					this_cpu_ptr ->
			init_timer
			add_timer()
				__mod_timer  ▻
			mod_timer
				__mod_timer
					internal_add_timer ->
			del_timer
			? setup_timer

			__run_timers
				timer_base_lock_expiry
				raw_spin_lock_irq

				collect_expired_timers
					pending_map
				__next_timer_interrupt
					next_pending_bucket
						pending_map
						time_before
				expire_timers
					detach_timer
					fn = timer->function;
					call_timer_fn
						trace_timer_expire_entry
						trace_timer_expire_exit timer_expire_exit
		__mod_timer
			running_timer
			internal_add_timer
				enqueue_timer
					trace_timer_start

=== IPC
		==> ./oo/conversion/cpu_mng/task/process/ipc/signals/src <==
		send_sig()
		signal()

		SIGINT
		SIGTSTP

		sys_kill
			siginfo
			kill_something_info
				kill_pid_info
					group_send_sig_info
						__group_send_sig_info
							send_signal **
								__send_signal
								__sigqueue_alloc
									sigpending
									sigaddset
									complete_signal
								signalfd_notify
								sigaddset
								complete_signal
		sys_signal
			do_sigaction lkm2
				k = &t->sighand->action[sig-1];

		linux/kernel/signal.c
		sigaction
		task_struct  blocked
		#!sh   ls -l
		-> scheduling

===== SysV IPC

		linux/ipc
		 ipc() system call

		message_queue

		sys calls
		sys_msgget()
		sys_msgctl()
		sys_msgrcv()

		structs
		msg_queue
		msg_msg
		msg_msgseg
		msg_sender
		msg_receiver
		msqid64_ds
		msqid_ds
		msg_setbuf

		uses wait_queue

		==> ./oo/conversion/cpu_mng/task/process/ipc/system_v_ipc/notes.txt <==
		5. IPC mechanisms
		5.1 Semaphores
		5.2 Message queues
		5.3 Shared Memory
		5.4 Linux IPC Primitives

		linux/ipc/sem.c
		syscalls:
		sys_semget()  ▻ ipc_lock  ▻ spin_lock
		sys_semctl()
		sys_semop()

		structs
		sem_array
		sem
		seminfo
		semid64_ds
		semid_ds
		sem_queue
		sembuf
		sem_undo

		shared_memory ./ipc/shm.c
		shmid_ds
		xx shm_segs
		shm_info

		system calls
		sys_shmget()
			ipcget
				ipc_namespace
				ipcget_new
					newseg
						ipc_addid
				ipcget_public
					ipc_check_perms
						shm_security
		shm_vm_ops

	raise_softirq	BH
		HI_SOFTIRQ .. NR_SOFTIRQS
		raise_softirq_irqoff ->

	BH
		DECLARE_TASKLET, tasklet_struct, tasklet_schedule (softirq context)
		DECLARE_WORK, work_struct (process context)
		softirq

	static void ldt_work_func(struct work_struct *work)
	{
	}

	static DECLARE_WORK(ldt_work, ldt_work_func);

	schedule_work(&ldt_work);

	tasklet_struct
		HI_SOFTIRQ, TASKLET_SOFTIRQ
		tasklet_init
		tasklet_schedule	ldt
			__tasklet_schedule
				tasklet_vec
				raise_softirq_irqoff(TASKLET_SOFTIRQ);
					__raise_softirq_irqoff
						or_softirq_pending(1UL << nr);
					wakeup_softirqd
		tasklet_kill
		task_queue
		#include <linux/tqueue.h>

		tq_struct

		softirq
		softirq_init
			open_softirq TASKLET_SOFTIRQ tasklet_action
				softirq_vec
				tasklet_action
					func
			open_softirq HI_SOFTIRQ tasklet_hi_action
				func

		ksoftirqd
			do_softirq  ▻

		DECLARE_TASK_QUEUE
		tq_timer
		tq_scheduler
		tq_immediate
		tq_disk
		tq_immediate
		IMMEDIATE_BH
		tq_disk

		void queue_task(struct tq_struct *task, task_queue *list);
		void run_task_queue(task_queue *list);

		Tasklets
		DECLARE_TASK_QUEUE(variablename);

		#include <linux/interrupt.h>

		tasklet_struct
		Tasklets


		tasklets are a special function that may be scheduled to run, in interrupt context, at a
		system-determined safe time.

		DECLARE_TASKLET(name, function, data)
		DECLARE_TASKLET_DISABLED(name, function, data)

		while (jiffies < j) schedule(); # not durable

		sys_nanosleep nanosleep
			hrtimer_nanosleep
				hrtimer_init_sleeper_on_stack
				hrtimer_set_expires_range_ns
				do_nanosleep
					hrtimer_sleeper_start_expires
						hrtimer_start_expires ->
							hrtimer_start_range_ns ->
					freezable_schedule
						schedule
				set_restart_fn(restart, hrtimer_nanosleep_restart);

		schedule_timeout_interruptible
			__set_current_state(TASK_INTERRUPTIBLE);
			schedule_timeout(timeout);

		sleep_on_timeout
			schedule_timeout->

		interruptible_sleep_on_timeout
			schedule_timeout  ▻

		Long:
			{	// together
				set_current_state(TASK_INTERRUPTIBLE);
				schedule_timeout (jit_delay*HZ);
			}
			ssleep
				msleep
					msecs_to_jiffies
					schedule_timeout_uninterruptible
						schedule_timeout
							timer_setup_on_stack
								__init_timer_on_stack
									init_timer_on_stack_key
										do_init_timer
											timer->function
							&process_timeout
								wake_up_process  ▻
							timer_list
							timer_setup
							x setup_timer
								init_timer
							__mod_timer  ▻
								internal_add_timer
									list_add_tail
							schedule

		common_interrupt x do_IRQ
			irq_exit

		x apic_timer_interrupt
			x smp_apic_timer_interrupt
				irq_exit
					invoke_softirq
					do_softirq
						local_softirq_pending
						__do_softirq
							softirq_vec
							action
							run_timer_softirq
								this_cpu_ptr
								timer_bases
								__run_timers ->

								process_timeout  ▻

	call_function_single_interrupt
		smp_call_function_single_interrupt
			irq_exit ->

	./arch/x86/include/asm/idtentry.h
		DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR,		sysvec_apic_timer_interrupt);

	DECLARE_IDTENTRY_SYSVEC
		asm_##func

	include/linux/tracepoint.h
			DEFINE_EVENT
				DECLARE_TRACE
					__DECLARE_TRACE
						__traceiter_##name
						trace_##name
							TP_ARGS
						register_trace_##name


	fec issue

		irqentry_nmi_enter
			rcu_nmi_enter
				trace_rcu_dyntick

		cpuidle_enter_state
			? asm_sysvec_apic_timer_interrupt (sysvec_apic_timer_interrupt)
				? sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c
					? __sysvec_apic_timer_interrupt
						hrtimer_interrupt ->
			rcu_idle_enter
				rcu_eqs_enter
					RCU extended quiescent
					trace_rcu_dyntick
			rcu_idle_exit
				rcu_eqs_exit
					rcu_dynticks_task_exit

			tick_irq_enter
				tick_nohz_irq_enter
					tick_nohz_stop_idle
						ts->idle_active = 0;
					tick_nohz_update_jiffies
						tick_do_update_jiffies64(now);
		irqentry_enter
			lockdep_hardirqs_off
			rcu_irq_enter
			trace_hardirqs_off_finish ->

		rcu_irq_exit
			rcu_nmi_exit
				trace_rcu_dyntick

		irq_enter_rcu ->

		DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt);

		DEFINE_IDTENTRY_SYSVEC
			run_sysvec_on_irqstack_cond
				ASM_CALL_SYSVEC
				call_on_irqstack_cond
					call_on_irqstack
						call_on_stack
					func


	x do_IRQ irq_work_interrupt reschedule_interrupt smp_apic_timer_interrupt

	native_smp_send_reschedule
	apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
		sysvec_reschedule_ipi
			5 ack_APIC_irq
			inc_irq_stat(irq_resched_count)
			scheduler_ipi
				preempt_fold_need_resched
					tif_need_resched_now
						test_thread_flag(TIF_NEED_RESCHED)
					set_preempt_need_resched ->

		irq_enter
			rcu_irq_enter
				lockdep_assert_irqs_disabled
				rcu_nmi_enter
			irq_enter_rcu
				__irq_enter_raw
					preempt_count_add(HARDIRQ_OFFSET);
				tick_irq_enter
				__irq_enter
		irq_exit
			irq_exit_rcu
				__irq_exit_rcu
					in_interrupt
						irq_count
							preempt_count
							softirq_count
				invoke_softirq
				in_hardirq
					hardirq_count
						preempt_count
						HARDIRQ_MASK
				wake_timersd RT
					timersd RT
					wake_up_process
				tick_irq_exit ->
			rcu_irq_exit
				rcu_nmi_exit
					rcu_dynticks_eqs_enter
			lockdep_hardirq_exit

		Short:
		#include <linux/delay.h>
		ndelay
			__ndelay
				__const_udelay  ▻
		udelay
			__udelay
				__const_udelay
					__delay
						cur_timer
		mdelay
			udelay

		#include <linux/param.h>
		HZ

		TIMER_IRQ

	... start_secondary
		setup_secondary_APIC_clock
		- setup_APIC_timer
			clockevents_register_device
				tick_check_new_device
					tick_setup_device
						tick_setup_periodic
							tick_set_periodic_handler
								dev->event_handler = tick_handle_periodic;
								dev->event_handler
								tick_handle_periodic ->

		irq0
		IRQF_SHARED
			hpet_time_init
				setup_default_timer_irq
			timer_interrupt ./arch/../kernel/time.c
				x do_timer_interrupt_hook
					global_clock_event->event_handler(global_clock_event

						tick_handle_periodic ->
				xx do_timer_interrupt
					mark_offset_tsc
					do_timer ->
		#include <linux/time.h>
		void do_gettimeofday(struct timeval *tv);
		void get_fast_time(struct timeval *tv);

		get_cycles
			rdtscll
				rdtsc
		sched_clock
			native_sched_clock
			rdtscll
			cycles_2_ns

		uptime_read_proc

		do_posix_clock_monotonic_gettime
			do_posix_clock_monotonic_gettime_parts

		setitimer() and getitimer() - not found

		current
		int in_interrupt();

		6. Timing Measurements
		      Hardware Clocks
		      The Linux Timekeeping Architecture
		      CPU's Time Sharing
		      Updating the Time and Date
		      Updating System Statistics
		      SoftwareTTimers
		      System Calls Related to Timing Measurements
		timer_interrupt  ▻
			xx do_timer_interrupt
				do_timer  ▻
					update_process_times ->
						update_one_process
							do_process_times

	kernel/sched.c
		sys_sched_getparam
			find_process_by_pid
				find_task_by_vpid
					find_task_by_pid_ns
						find_pid_ns
						pid_task
						task_struct
			p->rt_priority
		sys_sched_get_priority_max
		sys_sched_get_priority_min
		sys_sched_getscheduler
			security_task_getscheduler
		sys_sched_rr_get_interval
		sys_sched_setaffinity
			sched_setaffinity ->
				__sched_setaffinity task_struct
				... __set_cpus_allowed_ptr
					set_cpus_allowed ->
		sys_sched_getaffinity
			sched_getaffinity
				cpumask_and
				cpus_mask
				cpu_active_mask
		sys_sched_setparam
			do_sched_setscheduler
		sys_sched_setscheduler
			do_sched_setscheduler
		sys_sched_yield
			do_sched_yield
				yield_task
					yield_task_fair
						clear_buddies
				schedule

	kernel/signal.c
		sys_kill  ▻
		sys_pause
			schedule
		sys_restart_syscall
			&current_thread_info()->restart_block
		sys_rt_sigaction
			do_sigaction  ▻
		sys_rt_sigpending
			do_sigpending
		sys_rt_sigprocmask
			sigdelsetmask
		sys_rt_sigqueueinfo
			kill_proc_info
		sys_rt_sigsuspend
			schedule
		sys_rt_sigtimedwait
			schedule_timeout_interruptible
		sys_rt_tgsigqueueinfo
			do_rt_tgsigqueueinfo
				do_send_specific
					do_send_sig_info
		sys_tgkill
			do_tkill
				do_send_specific
		sys_tkill
			do_tkill

		delayed_work
			work_struct ▻
			timer_list ->
		cancel_delayed_work
}

=== CPU

	idle_setup
		"idle=poll" disables C-states
			IDLE_POLL
				cpu_idle_poll_ctrl


		==> ./oo/conversion/cpu_mng/task/connected_to.txt <==
		Scheduler
		Interrupts

		20. Program Execution
		      Executable Files
		      Executable Formats
		      Execution Domains
		      The exec Functions

		==> ./oo/conversion/cpu_mng/task/kinds.txt <==
		the idle thread(s),
		kernel threads,
		user process.

		==> ./oo/conversion/cpu_mng/task/notes.txt <==
		2. Process and Interrupt Management
		2.1 Task Structure and Process Table
		2.2 Creation and termination of tasks and kernel threads
		2.3 Linux Scheduler
		2.4 Linux linked list implementation
		2.5 Wait Queues
		2.6 Kernel Timers
		2.7 Bottom Halves
		2.8 Task Queues
		2.9 Tasklets
		2.10 Softirqs
		2.11 How System Calls Are Implemented on i386 Architecture?
		2.12 Atomic Operations
		2.13 Spinlocks, Read-write Spinlocks and Big-Reader Spinlocks
		2.14 Semaphores and read/write Semaphores
		2.15 Kernel Support for Loading Modules

		3. Processes
		      Processes, Lightweight Processes, and Threads
		      Process Descriptor
		      Process Switch
		      Creating Processes
		      Destroying Processes

		pid_t

		== current

		== task_struct
			mm_struct
				vm_area_struct
					vm_start

	union thread_union init_thread_union __init_task_data = { INIT_THREAD_INFO(init_task) };

	thread_union

		trvx(current_stack_pointer & (THREAD_SIZE - 1));

		thread_info asm/thread_info.h
			task_struct
			exec_domain
			current_thread_info
				return (struct thread_info *) (current_stack_pointer & ~(THREAD_SIZE - 1));
				current_stack_pointer
					asm("esp")
		stack

		scheduler
		current <include/asm/current.h>

			task_struct* current

		Thread in Linux are just processes that share VM

		kernel_thread
			CLONE_VM CLONE_UNTRACED
			do_fork

		add_wait_queue() kernel function
		remove_wait_queue()  kernel function
		sleep_on()
		linux/kernel/fork.c
	IPC
		fsnotify_init
		dnotify_init	Directory notifications for Linux. - obsoleted by inotify
		fanotify_user_setup
		inotify_user_setup

		sys_inotify_init  ▻

		sys_fanotify_init	fs/notify/fanotify/fanotify_user.c	lkm2
			FAN_ALL_EVENTS
			fanotify_fops
		fsnotify_init
		ipc/mqueue.c
			sys_mq_open
				do_create
					vfs_create  ▻
				do_open
			sys_mq_getsetattr
			sys_mq_notify
				alloc_skb
				netlink_attachskb
			sys_mq_timedreceive
				msg_get
				pipelined_receive
					msg_insert
				store_msg
			sys_mq_timedsend
				load_msg
				pipelined_send
					list_del
			sys_mq_unlink
				vfs_unlink
		ipc/msg.c
			sys_msgctl
			sys_msgget
			sys_msgrcv
				do_msgrcv
					store_msg
			sys_msgsnd
				do_msgsnd
					pipelined_send
		ipc/sem.c
			sys_semctl
			sys_semget
			sys_semop
			sys_semtimedop
		ipc/shm.c
			sys_shmat	lkm2
				do_shmat
					do_mmap  ▻	lkm2
			sys_shmctl
			sys_shmdt
				do_munmap  ▻
			sys_shmget  ▻

		19. Process Communication
		      Pipes
		      FIFOs
		      System V IPC

		pipe()
		open() named pipe
		uses wait queue, signals
		pipe_read
		pipe_readv
			fs_table
				pipe_max_size

		mkfifo

		10. Signals
		      The Role of Signals
		      Generating a Signal
		      Delivering a Signal
		      System Calls Related to Signal Handling

	get_cpu_var
		preempt_disable
		this_cpu_ptr

	put_cpu_var
		preempt_enable

	get_cpu_ptr
		get_cpu_ptr() disables preemption and therefore migration
		preempt_disable include/linux/preempt.h
			inc_preempt_count
				add_preempt_count
					preempt_count
						pcpu_hot.preempt_count
						current_thread_info()->preempt_count
		this_cpu_ptr raw_cpu_ptr
				__verify_pcpu_ptr
				arch_raw_cpu_ptr
					this_cpu_off
					raw_smp_processor_id ->
	put_cpu_ptr
		preempt_enable

	proc_pid_sched_operations
		sched_open
			sched_show
				proc_sched_show_task
					/proc/self/sched
						policy
						prio
					wait_sum
	do_task_stat
		task_prio
			return p->prio - MAX_RT_PRIO;
		task_nice ->
		...
		task->rt_priority
		task->policy

	ps
		openproc
			simple_readproc
				stat2proc
		PIDS_PRIORITY stat: priority
		finalize_stacks
			pr_priority

	is_migration_disabled

==== affinity
	see also smp_affinity_enable
		irq_update_affinity_hint
			__irq_apply_affinity_hint




		map_queues_v2_hw
			blk_mq_queue_map
			irq_get_affinity_mask
				irq_data_get_affinity_mask
					desc->irq_common_data.affinity;

		irq_data_update_affinity
			cpumask_copy(d->common->affinity, m);

		apic_update_irq_cfg
			irq_data_update_effective_affinity
				effective_affinity

		mlx4_en_activate_cq
			irq_get_effective_affinity_mask
				irq_data_get_effective_affinity_mask

		irq_thread_check_affinity ->

		"rq_affinity"
			queue_rq_affinity_store

		pci_alloc_irq_vectors_affinity->

tools/testing/selftests/cgroup/test_cpuset_prs.sh

	run_state_test
		set_ctrl_state_noerr
		set_ctrl_state
			write_cpu_online
				/sys/devices/system/cpu/cpu${CPU}/online 
	test_isolated
		test_partition
			cpuset.cpus.partition
		test_effective_cpus
			cpuset.cpus.effective
	test_inotify
		wait_inotify
		set_ctrl_state
		check_cgroup_states
		online_cpus
			write_cpu_online

kernel/cgroup/cpuset.c

		cpuset_cgrp_subsys
			dfl_files
				cpuset_common_seq_show
					spin_lock_irq(&callback_lock);

				cpuset_write_resmask ->
				sched_partition_write
					update_prstate
						partition_xcpus_newstate

	       for_each_cpu(cpu, enable_mask)
		       timers_prepare_cpu(cpu);

	       for_each_cpu(cpu, disable_mask)
		       timers_resettle_from_cpu(cpu);

		mce_timer_fn

		grep mce_timer_fn /proc/timer_list

		gqip

	watchdog_cpumask is initialized in lockup_detector_init() from housekeeping_cpumask(HK_TYPE_TIMER).

	lockup_detector_reconfigure() utilizes watchdog_cpumask via __lockup_detector_reconfigure().

	kernel_init
	kernel_init_freeable
	do_one_initcall
	acpi_init
	acpi_scan_init
	acpi_bus_scan
	acpi_bus_attach
	acpi_dev_for_each_child
	device_for_each_child
	? acpi_dev_for_one_check
	acpi_bus_attach
	device_for_each_child
	? acpi_dev_for_one_check
	acpi_bus_attach
	acpi_create_platform_device
	acpi_dev_get_resources
	acpi_walk_resources
	? acpi_dev_process_resource
	acpi_walk_resource_buffer
	? acpi_dev_process_resource
	acpi_dev_process_resource
	acpi_dev_resource_interrupt
	? acpi_dev_resource_address_space
	acpi_dev_get_irqresource
	? lock_acquire
	acpi_register_gsi_ioapic
	mp_map_pin_to_irq
	alloc_isa_irq_from_domain
	__irq_domain_alloc_irqs
	irq_domain_alloc_irqs_locked
	dump_stack_lvl

	really_probe
	platform_probe
	ged_probe
	acpi_walk_resources
	acpi_walk_resource_buffer
	acpi_ged_request_interrupt
	request_threaded_irq
	__setup_irq
	irq_startup
	irq_setup_affinity
	irq_do_set_affinity
		ioapic_set_affinity
			parent->chip->irq_set_affinity
			apic_set_affinity

	mp_register_ioapic

	mp_ioapic_irqdomain_ops
		mp_irqdomain_alloc
			ioapic_chip
				->ioapic_set_affinity

		cpus_timer_store
		isolate
			resettle_all_timers
				tick_cpu_dying tick_nohz_handler
				hrtimers_cpu_evict

		syscore_suspend

		timekeeping_syscore_ops
			timekeeping_suspend
				tick_suspend
					tick_suspend_local
					tick_suspend_broadcast
						clockevents_shutdown

		tick_nohz_full_stop_tick
			tick_nohz_next_event
			tick_nohz_stop_tick
			tick_nohz_retain_tick

		tick_nohz_handler ->

		cpu_startup_entry
			do_idle ->
				tick_nohz_idle_enter ->
					[TS_FLAG_INIDLE] = 1
				tick_nohz_idle_exit ->
					WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
					[TS_FLAG_INIDLE] = 0

					housekeeping_update
						timers_prepare_cpu
						hrtimers_prepare_cpu
						hrtimers_cpu_evict
							smp_call_function_single ->
							retrigger_next_event
						timers_dead_cpu ->

						  ipi_send_cpu: cpu=11 callsite=irq_work_queue_on+0x109/0x120 callback=nohz_full_kick_func+0x0/0x10
	<...>-111     [009] d...3.. 94815.741420: ipi_send_cpu: cpu=11 callsite=check_preempt_curr+0x33/0x70 callback=0x0

=== Interrupts

			fixup_irqs
				irq_migrate_all_off_this_cpu
					for_each_active_irq
						irq_get_next_irq
							irq_find_at_or_after
								sparse_irqs
					irq_domain_deactivate_irq
					raw_spin_lock(&desc->lock);
					migrate_one_irq
						irq_datairq_data
						irq_chip
						irqd_irq_masked
						irq_mask: mask_ioapic_irq  pci_irq_mask_msix < pci_msix_template
						irq_desc_get_irq_data
							irq_data
						irq_force_complete_move ->
						irq_fixup_move_pending
						irq_desc_get_pending_mask
							pending_mask
						irq_data_get_affinity_mask ->
						irq_do_set_affinity ->

							housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
							irq_common_data.affinity
						irq_needs_fixup
						chip->irq_mask
						irqd_affinity_is_managed
						irqd_set_managed_shutdown
							IRQD_MANAGED_SHUTDOWN
						irq_shutdown_and_deactivate ->

	arch/x86/include/asm/trace/irq_vectors.h
		DECLARE_EVENT_CLASS(x86_irq_vector,
			"vector="
		DEFINE_IRQ_VECTOR_EVENT(irq_work);
		DEFINE_IRQ_VECTOR_EVENT(local_timer);
			##_entry
			##_exit
			DEFINE_EVENT_FN
				DECLARE_TRACE ->
			trace_local_timer_entry
			trace_local_timer_exit

	arch/x86/include/asm/idtentry.h
	arch/x86/kernel/apic/apic.c
		sysvec_apic_timer_interrupt
			"local_timer"
			DEFINE_IDTENTRY_SYSVEC
			trace_local_timer_entry(LOCAL_TIMER_VECTOR);
				"local_timer_entry: vector=236"
				trace_intel_irq_entry
					osnoise_trace_irq_entry	 ->
			local_apic_timer_interrupt
			trace_local_timer_exit(LOCAL_TIMER_VECTOR);
				trace_intel_irq_exit
					osnoise_trace_irq_exit ->
			set_irq_regs


	handle_nested_irq
		thread_fn
		wake_threads_waitq

	IRQ_WORK_INIT*
		nohz_full_kick_func task_work_set_notify_irq rcu_iw_handler late_wakeup_func
		call_rcu_tasks_iw_wakeup wake_up_klogd_work_func irq_sim_handle_irq
		perf_pending_disable irq_work_wake

	early_idt_handler_array
	

	__irq_set_handler
		__irq_do_set_handler

	irq_work_init_threads
		irqwork_threads "irq_work/%u"
			run_irq_workd
				irq_work_run_list
					irq_work_single
						work->func
							wake_up_klogd_work_func->
						rcuwait_wake_up
							rcu_dereference
		smpboot_register_percpu_thread''

	
	idt_setup_traps
		idt_setup_from_table

	idt_setup_apic_and_irq_gates
		idt_setup_from_table
			idt_init_desc
				offset_high, offset_low = addr
			vector
			write_idt_entry
				native_write_idt_entry
					idt[entry] = gate
		apic_idts
			reschedule_interrupt
			call_function_interrupt
			call_function_single_interrupt
			INTG(LOCAL_TIMER_VECTOR,		asm_sysvec_apic_timer_interrupt),


		irq_action
		irqaction

		{
		    save_flags(flags); // better spin_lock_irqsave
		    cli();

		    /* This code runs with interrupts disabled */

		    restore_flags(flags); // instead sti;
		}

		linux/interrupt.h

		free_irq

		request_irq kernel/irq/manage.c
			request_threaded_irq  ▻
			setup_irq
				__setup_irq
					setup_irq_thread
						&irq_thread
							irq_wait_for_interrupt
								irq_thread_check_affinity
									irq_data_get_effective_affinity_mask
									set_cpus_allowed_ptr
								test_and_clear_bit
								IRQTF_RUNTHREAD,
							irq_thread_fn
								thread_fn
						get_task_struct
					desc->action = irqaction
					irq_desc
					wake_up_and_wait_for_irq_thread_ready
						wake_up_process
					register_irq_proc
						/proc/irq/
					register_handler_proc
						/proc/irq/1234/handler/

		...
			handle_irq_event
				handle_irq_event_percpu
					__handle_irq_event_percpu
						__irq_wake_thread
							test_and_set_bit
							IRQTF_RUNTHREAD
							wake_up_process(action->thread);

		/proc/interrupts
			proc_interrupts_init
				int_seq_ops
					show_interrupts
						desc = irq_to_desc(i);
							sparse_irqs
						arch_show_interrupts
							"Non-maskable interrupts" ...
							irq_stats
							for_each_online_cpu
								for_each_cpu
									for_each_set_bit

		/proc/stat

		4. Interrupts and Exceptions
		      The Role of Interrupt Signals
		      Interrupts and Exceptions
		      Nested Execution of Exception and Interrupt Handlers
		      Initializing the Interrupt Descriptor Table
		      Exception Handling
		      Interrupt Handling
		      Softirqs, Tasklets, and Bottom Halves
		      Returning from Interrupts and Exceptions

	IRQ_PER_CPU IRQD_PER_CPU
		irq_settings_set_per_cpu
		irq_settings_is_per_cpu irq_is_percpu irqd_is_per_cpu

	irqd_can_balance->

	include/linux/irqdesc.h
		irq_is_*:
			irq_check_status_bit
				irq_to_desc
		irq_balancing_disabled
			IRQ_NO_BALANCING_MASK
				IRQ_PER_CPU | IRQ_NO_BALANCING
		irq_is_percpu
			IRQ_PER_CPU
		irq_is_percpu_devid
			IRQ_PER_CPU_DEVID
	kernel/irq/settings.h
		irq_desc:
			_IRQ_NO_BALANCING
				irq_settings_set_no_balancing
					< __setup_irq
				IRQF_NOBALANCING -> irq_settings_has_no_balance_set -> IRQD_NO_BALANCING
					< irq_modify_status
			_IRQ_PER_CPU IRQ_PER_CPU:
				irq_settings_set_per_cpu
					< __setup_irq

				irq_settings_is_per_cpu -> IRQD_PER_CPU
					<
						irq_modify_status
							IRQD_PER_CPU
						?__irq_get_desc_lock
						kstat_irqs_desc
							< kstat_irqs
						try_one_irq

		irq_settings_is_per_cpu_devid
			_IRQ_PER_CPU_DEVID IRQ_PER_CPU_DEVID

	irq_modify_status->

	irqd_is_per_cpu
	<
		migrate_one_irq
		irq_move_masked_irq
		show_interrupts

		IRQD_PER_CPU

	IRQF_PERCPU IRQD_PER_CPU
	__irq_can_set_affinity

		IDT	Interrupt Descriptor Table
		show_interrupts

		setup_vector_irq
			__setup_vector_irq
				vector_irq

		x do_IRQ
			unsigned vector = ~regs->orig_ax;
			irq = __this_cpu_read(vector_irq[vector]);
			irq_desc
			x __do_IRQ
				x handle_IRQ_event
					timer_interrupt  ▻
			handle_irq
				desc->handle_irq
				x irq_handler

		linux/include/asm/irq.h  :

		disable_irq
		disable_irq_nosyn
		enable_irq

		Restrictions:
		A handler can't transfer data to or from user space, because it doesn't execute in the context of a process.
		other than GFP_ATOMIC, or locking a semaphore.
		Finally, handlers cannot call schedule.

		void short_interrupt(int irq, void *dev_id, struct pt_regs *regs)
		{
		    struct timeval tv;
		    int written;

		    do_gettimeofday(&tv);

		    /* Write a 16-byte record. Assume PAGE_SIZE is a multiple of 16 */
		    written = sprintf((char *)short_head,"%08u.%06u\n",
				      (int)(tv.tv_sec % 100000000), (int)(tv.tv_usec));
		    short_incr_bp(&short_head, written);
		    wake_up_interruptible(&short_queue); /* wake any reading process */
		}

		not for SNP

		unsigned long flags;

		save_flags(flags);
		cli();
		/* critical code */
		restore_flags(flags);

		global_bh_lock
		init_bh
		remove_bh
		mark_bh

		BH

		devm_irq_alloc_desc
			...
				__devm_irq_alloc_descs
					devm_irq_desc_release

						irq_free_desc ->
		<linux/interrupt.h>

			irq_free_desc
				irq_free_descs
					free_desc
						delete_irq_desc

		2. Process and Interrupt Management
		2.1 Task Structure and Process Table
		2.2 Creation and termination of tasks and kernel threads
		2.3 Linux Scheduler
		2.4 Linux linked list implementation
		2.5 Wait Queues
		2.6 Kernel Timers
		2.7 Bottom Halves
		2.8 Task Queues
		2.9 Tasklets
		2.10 Softirqs
		2.11 How System Calls Are Implemented on i386 Architecture?
		2.12 Atomic Operations
		2.13 Spinlocks, Read-write Spinlocks and Big-Reader Spinlocks
		2.14 Semaphores and read/write Semaphores
		2.15 Kernel Support for Loading Modules
	PCI
		examples
			snd_intel8x0, snd_intel8x0_probe
			hl_device_init
		pci_error_handlers
			pci_ers_result_t <- pci_channel_state_t
			pci_ers_result
		AER - Advanced Error Reporting
		BAR - Base Address Registers
		PCI_CAP_ID_PCIX
		pcie_port_service_driver
			PCIE_PORT_SERVICE_PME	Power Management Event
			PCIE_PORT_SERVICE_AER	Advanced Error Reporting
			PCIE_PORT_SERVICE_HP	Native Hotplug
			PCIE_PORT_SERVICE_DPC	Downstream Port Containment
			PCIE_PORT_SERVICE_BWNOTIF	Bandwidth notification
		PMC - Power Management Controller
		ACPI - Advanced Configuration and Power Interface
		pci_dev_id
			PCI_DEVID
				PCI_BUS_NUM 15:8
				PCI_DEVFN(slot, func)
					PCI_SLOT 7:3
					PCI_FUNC 2:0
		__pci_read_base
			pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
		pci_setup_device
			pci_hdr_type
				PCI_HEADER_TYPE
			PCI_HEADER_TYPE_NORMAL
			PCI_HEADER_TYPE_BRIDGE
			PCI_HEADER_TYPE_CARDBUS

			pci_request_regions
				pci_request_region
					request_region  ▻
					request_mem_region

				pci_request_selected_regions ->
			pci_release_regions
				pci_release_selected_regions
					pci_release_region
						release_region
							ioport_resource
							__release_region >
						release_mem_region
			pcim_iomap_regions_request_all 16 uses
				~
				pci_request_selected_regions
					__pci_request_selected_regions
						__pci_request_region ->
							request_region
								ioport_resource
								__request_region kernel/resource.c
									alloc_resource
									__request_resource
										&root->child
										new->parent = root
							request_mem_region
								iomem_resource
								__request_region
									__request_resource
							__request_mem_region
								&iomem_resource
								__request_region->
						-EBUSY
				pcim_iomap_regions
					pci_request_region
						__pci_request_region
					pcim_iomap
						pcim_iomap_table
							devres_find
							pcim_iomap_release
						pci_iomap
							pci_iomap_range
								ioremap
			1
				pci_enable_device 904 uses
				pci_set_dma_mask
				pci_request_regions
				pcim_iomap / pci_iomap / pci_ioremap_bar
				pci_iounmap
					iounmap
				pci_release_regions 616 uses
				pci_disable_device
			2
				pcim_iomap_regions 115 uses
				pcim_iomap_table
				pci_set_master 639 uese
				pcim_iounmap_regions 16
			3
				pci_select_bars
				pci_request_selected_regions 59 uses
				pci_ioremap_bar/ devm_ioremap
				pci_iounmap
				pci_release_selected_regions 54 uses
			4
				pci_request_mem_regions
				ioremap pci_iomap
				pci_release_mem_regions

			pcim_iounmap_regions (pcim_iomap_regions)
				pcim_iounmap
				pci_release_region
			pci_enable_device_mem 73 uses
		pci_register_driver ->
		pci_set_pcie_reset_state
		pcie_capability_read_word       179
		pci_is_pcie     123
		pcie_capability_write_word      87
		pci_pcie_type   83
		pci_disable_pcie_error_reporting	55
		pci_enable_pcie_error_reporting 48
		pcie_capability_read_dword      48
		pcie_capability_clear_and_set_word      38
		pcie_set_readrq 34
		pcie_get_mps    23
		pcie2_write32   22
		pci_pcie_cap    21
		pcie_flr	21
		pcie_print_link_status  20
		pcie_get_readrq 20
		pcie_capability_clear_word      18
		pcie_bus_configure_settings     18
		bcma_drv_pcie2  17
		pcie_aspm_enabled
			pcie_aspm_get_link
			aspm_enabled
		pci_dev_msi_enabled
		Endpoint Core
			EPC  Endpoint Controller
				include/linux/pci-epc.h
				pci_epc_create
				devm_pci_epc_create
			EPF PCI Endpoint Function
				include/linux/pci-epf.h
				pci_epf_create
				pci_epf_test_init
					pci_epc_set_bar	include/linux/pci-epc.h

		a

		mlx5_irq_table_create
			PCI_IRQ_MSIX
			pci_alloc_irq_vectors

		PCI_CAP_ID_MSIX
		PCI_CAP_ID_EXP
		pci_alloc_irq_vectors
			masks[i].is_managed = 0 ?
			affd = 0
			pci_alloc_irq_vectors_affinity
				PCI_IRQ_MSI
					__pci_enable_msi_range
						irq_calc_affinity_vectors
						msi_capability_init
							irq_affinity *affd
							irq_affinity_desc
							irq_create_affinity_masks ->
							arch_setup_msi_irqs
							msi_setup_msi_desc
								irq_affinity_desc *masks
								desc.affinity = masks;
								msi_insert_msi_desc ->
				PCI_IRQ_MSIX
					__pci_enable_msix_range
						x __pci_enable_msix
						msix_capability_init
							msix_setup_interrupts
								irq_create_affinity_masks ->
								irq_affinity_desc <- irq_create_affinity_masks ->
								msix_setup_msi_descs
									msi_desc.affinity <- irq_affinity_desc
									msi_desc
									desc.affinity = masks ? curmsk : NULL;
									msix_prepare_msi_desc
									msi_insert_msi_desc
										msi_domain_insert_msi_desc
											msi_alloc_desc
											msi_insert_desc
												msi: dev_msi_info
												msi.data: msi_device_data
												__domains: msi_dev_domain
												store: xarray
												dev->msi.data->__domains[domid].store
												msi_device_data
				PCI_IRQ_INTX
					nvecs = 1
					irq_create_affinity_masks
						irq_affinity_desc <- irq_affinity
						group_cpus_evenly ->
						cpumask_copy(&masks[curvec].mask, ...);
						masks[i].is_managed = 1;
					pci_intx
						PCI_COMMAND_INTX_DISABLE
		pci_free_irq_vectors
			pci_disable_msix ->
			pci_disable_msi
				pci_msi_shutdown
				free_msi_irqs
					// when pci_free_irq was called
					// free_msi_irqs: BUG_ON(irq_has_action(entry->irq + i));



		pci_request_irq
			<-
				drivers/infiniband/hw/hfi1/msix.c msix_request_irq
				drivers/infiniband/hw/qib/
					qib_setup_6120_interrupt
					qib_setup_7220_interrupt
					qib_setup_7322_interrupt
				zoran_probe
				drivers/net/ethernet/cavium/thunder/thunder_bgx.c
					bgx_register_intr
				queue_request_irq (nvme)
					cq_vector
				drivers/ptp/ptp_ocp.c
					ptp_ocp_register_ext
						irq_vec
				sound/soc/intel/avs/core.c
					avs_hdac_acquire_irq

			pci_irq_vector
				x
				for_each_pci_msi_entry
				first_pci_msi_entry
					first_msi_entry
						list_first_entry
			request_threaded_irq ->
		pci_free_irq
			free_irq
				__free_irq
					irq_shutdown
						irq_state_set_disabled
							IRQD_IRQ_DISABLED
							irqd_set
								__irqd_to_state
						__irq_disable
							irq_disable
						irq_state_clr_started
							irqd_clear
								__irqd_to_state
					irq_release_resources
		pcie_pme_init
			pcie_pme_driver
				pcie_pme_probe
			pcie_port_service_register
		pcie_bandwidth_notification_init
			pcie_port_service_register
		depricated
			pci_enable_msi, use pci_alloc_irq_vectors
				pci_enable_msi_block
					msi_capability_init
						arch_setup_msi_irqs  ▻
			pci_disable_msi -> , use pci_free_irq_vectors
			pci_enable_msix_exact
				pci_enable_msix_range use pci_alloc_irq_vectors
					__pci_enable_msix_range
						msix_enabled
			pci_disable_msix
				pci_msix_shutdown
				free_msi_irqs
		pci_read_config_dword
			pci_bus_read_config_dword
				pci_bus
				pci_ops
					pci_root_ops
		core
			pci_pcbios_init
				raw_pci_ops = pci_find_bios
					pci_bios_access
						pci_bios_read
						pci_bios_write
			pci
				pci_legacy_init
					pcibios_scan_root
				acpi_pci_root_add
					pci_acpi_scan_root
						pcibios_scan_root
							pci_scan_bus_parented
								pci_create_bus
							pci_root_ops
								pci_read
									raw_pci_ops->read
									?? raw_pci_read
								pci_write
									raw_pci_ops->write
				pcibios_init
				pci_init
					pci_dev
					pci_get_device
						pci_get_subsys
							pci_get_dev_by_id
								match_pci_dev_by_id
									pci_match_one_device
										pci_device_id
								bus_find_device
								to_pci_dev
					pci_fixup_device
			pci_bus_read_dev_vendor_id
				pci_bus_generic_read_dev_vendor_id
					pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l)
			pci_status_get_and_clear_errors
				ret = pci_read_config_word(pdev, PCI_STATUS, &status);
		PCI Hotplug
		pci_do_scan_bus
			pci_scan_child_bus
			pci_bus_add_devices
				pci_bus_add_device
					pci_devices

		acpi_pci_root_init
			acpi_bus_register_driver
				acpi_bus_drivers
				acpi_driver_attach
					acpi_bus_driver_init
						add  ▻
			acpi_pci_root_driver
				acpi_pci_root_add  ▻
				acpi_pci_root_start
					acpi_pci_roots
					pci_bus_add_devices

		? pci_scan_bus
				pci_fixup_device
					pci_do_fixups
		pci_driver_init
			bus_register  ▻
			pci_bus_type
				pci_hotplug

			pci_set_consistent_dma_mask

			DMA_28BIT_MASK ?

			linux/Documentation/pci.txt

			pci_driver
				device_driver
				pci_register_driver  (pci_module_init)
					__pci_register_driver
						driver_register  ▻
						pci_create_newid_file
							sysfs_create_file  ▻
				pci_unregister_driver

				??
				pci_bus_type
					pci_device_probe
						__pci_device_probe
							pci_match_device
								pci_match_id
									pci_match_one_device
							pci_call_probe
								wq_domain_mask
								local_pci_probe
									probe()
										 skeleton_probe
						pci_dev_get
							get_device
						pci_dev_put
							put_device
								kobject_put
									kref_put
									\kobject_release
										kobject_cleanup
											__kobject_del
												< kobject_del
											t->release

				pci_scan_slot
					pci_scan_single_device
						pci_scan_device
							pci_setup_device
								pci_read_irq
									pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
									pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
									dev->irq = irq
								pci_subsystem_ids
									PCI_SUBSYSTEM_VENDOR_ID
									PCI_SUBSYSTEM_ID
								pci_read_bases -reading resources patch
									PCI_BASE_ADDRESS_0
									pci_size
						pci_device_add
							pci_init_capabilities ->
								pci_pm_init
								pci_iov_init  ▻
							list_add_tail(&dev->bus_list, &bus->devices);

			CONFIG_PCI_IOV
			pci_assign_resource
				pci_assign_bus_resource
					allocate_resource
						find_resource
					pcibios_update_resource
			struct pci_dev (before pci_physfn)
				bus_list
				msi_enabled
				msix_enabled
				aer_cap, pcie_mpss <- key
				-> pci_driver
				pcie_capability_read_dword(pcid, PCI_EXP_LNKCAP, &lnk_cap);
				trlvx(lnk_cap);
				trlvx(lnk_cap & PCI_EXP_LNKCAP_LBNC);
				trlvx(pcid->class);
				trlvx(pcid->cfg_size);
				for (i = 0; i < DEVICE_COUNT_RESOURCE; ++i) {
					if (!pci_resource_flags(pcid, i))
						continue;
					trlvd(i);
					trvs(pcid->resource[i].name);
					trlvx(pci_resource_start(pcid, i));
					trlvx(pci_resource_len(pcid, i));
					trlvx(pci_resource_flags(pcid, i));
					//trlvx(pci_resource_flags(pcid, i) & IORESOURCE_TYPE_BITS);
					trvdnz((pci_resource_flags(pcid, i) & IORESOURCE_IO) > 0);
					trvdnz((pci_resource_flags(pcid, i) & IORESOURCE_MEM) > 0);
				}

			CONFIG_HOTPLUG
				pci_insert_device
			pbus_assign_resources
				pbus_assign_resources_sorted
					pci_assign_resource  ▻

			pcibios_assign_resources 2.4-sh
				pci_assign_resource  ▻

			proc
				show_dev_config  (/proc/pci)
				show_device
				pci_proc_attach_device

		do_pci_enable_device
		device_ktype
			device_release
				devres_release_all
					< really_probe
					< __device_release_driver
					release_nodes
					release
						pcim_release
							pci_disable_device
		pcim_enable_device
			pci_devres
			get_pci_dr
				pcim_release
				devres_find
				devres_alloc
					__devres_alloc_node
						alloc_dr ->
				devres_get
			pci_enable_device
				__pci_enable_device
					pci_enable_device_bars
						pci_set_power_state
						pcibios_enable_device
							pcibios_enable_resources
								pci_read_config_word(dev, PCI_COMMAND, &cmd);
									pci_bus_read_config_word
										PCI_OP_READ
											bus->ops->read
							pcibios_enable_irq
								pirq_enable_irq
									"can't find IRQ for PCI INT"
					pci_fixup_device
		pci_disable_device
			do_pci_disable_device

		Linux
			basic
				linux/config.h _LINUX_CONFIG_H
					include/linux/autoconf.h AUTOCONF_INCLUDED
				linux/version.h
					LINUX_VERSION_CODE
					KERNEL_VERSION
				linux/module.h _LINUX_MODULE_H
				linux/kernel.h _LINUX_KERNEL_H
			irq
			linux/sched.h _LINUX_SCHED_H
				task_struct

			readl readw readb

		lspci

		pcitweak

		cat /proc/bus/pci/devices

		od -Ax -t x1 /proc/bus/pci/*/*

		PCI Addressing

		Configuration Space

		#include <linux/pci.h>

		pci_bus
			struct pci_dev ->
		struct pci_device_id;

		pci_dev
			x pci_find_device
			x pci_find_class

		pci_read_config_word
		pci_write_config_word

		kobject_hotplug
			hotplug_path

		pci_present obsolete
		readl readw readb

		device drivers

		13. Managing I/O Devices
		      I/O Architecture
		      Device Files
		      Device Drivers
		      Block Device Drivers
		      Character Device Drivers

		firmware
			request_firmware
}

==== irq affinity

	irq_affinity_setup "irqaffinity"
		irq_default_affinity

	irq_default_affinity
		<
			irq_create_affinity_masks is_managed
				<
					devm_platform_get_irqs_affinity ->
					1 pci_alloc_irq_vectors_affinity
						<
							nvme_setup_irqs
							pci_alloc_irq_vectors
							drivers/scsi/
								_base_alloc_irq_vectors
								be2iscsi_enable_msix
								csio_enable_msix
								interrupt_preinit_v3_hw
								__megasas_alloc_irq_vectors
								mpi3mr_setup_isr
								pm8001_setup_msix
								qla24xx_enable_msix
							drivers/virtio/virtio_pci_common.c
								vp_request_msix_vectors
							drivers/net/ethernet/wangxun/
								wx_acquire_msix_vectors
					msi_capability_init
						< __pci_enable_msi_range
							< pci_alloc_irq_vectors_affinity
					msix_setup_interrupts
						msix_capability_init
							< __pci_enable_msix_range
								pci_enable_msix_range
								pci_alloc_irq_vectors_affinity
			irq_affinity_setup
			init_irq_default_affinity
			desc_smp_init
			irq_setup_affinity
			default_affinity_show

	irq_startup
	> irq_setup_affinity
		irq_default_affinity
		irq_desc
		__irq_can_set_affinity
		irq_do_set_affinity
			optimistic tmp_mask = mask & housekeeping_cpumask(HK_TYPE_MANAGED_IRQ) & cpu_online_mask

			irqd_affinity_is_managed
				__irqd_to_state(d)
				IRQD_AFFINITY_MANAGED;
			hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
			chip->irq_set_affinity
				msi_set_affinity
					apic_set_affinity
				apic_set_affinity
					assign_managed_vector
						vector_matrix
						irq_data_get_affinity_mask
						vector_searchmask
						irq_matrix_alloc_managed -> vector
							matrix_find_best_cpu_managed
						apic_update_vector
							move_in_progress
							irq_matrix_free
								alloc_map
							vector_irq
						apic_update_irq_cfg ->
				msi_domain_set_affinity
					irq_chip_write_msi_msg
			irq_validate_effective_affinity
				irq_data_get_effective_affinity_mask->
			irq_set_thread_affinity
				irqaction
				IRQTF_AFFINITY,
			cpumask_copy(desc->irq_common_data.affinity, mask);

	kernel_init
		kernel_init_freeable
			do_basic_setup
				init_irq_proc
					register_default_affinity_proc
						/proc/irq/default_smp_affinity
						default_affinity_show
							irq_default_affinity
						default_affinity_write
							cpu_online_mask
			smp_prepare_cpus
				set_cpu_present

	/proc/irq/*/effective_affinity  /proc/irq/*/effective_affinity_list

	"effective_affinity_list"

	"smp_affinity"
	irq_affinity_proc_ops
		irq_affinity_proc_open
			irq_affinity_proc_show
				show_irq_affinity
					irq_desc::irq_data
					mask = desc->irq_common_data.affinity;
					mask = irq_data_get_effective_affinity_mask

		irq_affinity_proc_write
			write_irq_affinity ->

	irq_affinity_list_proc_ops
		irq_affinity_list_proc_open
			irq_affinity_list_proc_show
				show_irq_affinity ->
		irq_affinity_list_proc_write
			write_irq_affinity
				cpu_online_mask
				irq_can_set_affinity_usr
					!irqd_affinity_is_managed
				irq_select_affinity_usr
				irq_set_affinity ->

	irq_common_data::msi_desc::irq_affinity_desc::mask
	irq_common_data::msi_desc::irq_affinity_desc::is_managed

	start_kernel
		early_irq_init
			irq_desc
				desc_set_defaults
				irq_data
					irq
					->irq_common_data
					->irq_chip
					->irq_domain
				irq_common_data
					affinity
					msi_desc
						msi_msg
						irq_affinity_desc
							cpumask	mask;
			arch_early_irq_init
				x86_vector_domain_ops
					x86_vector_alloc_irqs
						assign_irq_vector_policy
							reserve_managed_vector
								irq_matrix_reserve_managed
						lapic_controller
							->apic_set_affinity ->
						irqd_set_single_target
							IRQD_SINGLE_TARGET

	irq_get_affinity_mask    14
	irq_can_set_affinity     14
		<
			dpaa_set_portal_irq_affinity
			tick_check_percpu
		__irq_can_set_affinity
			irqd_can_balance
				!IRQD_PER_CPU & !IRQD_NO_BALANCING
	pci_alloc_irq_vectors_affinity   15
	irq_data_get_effective_affinity_mask     17
		irq_desc::irq_data::irq_common_data::effective_affinity;
	irq_affinity_desc	33
	irq_set_affinity_notifier	30
		irq_affinity_notify      41
	irq_chip_set_affinity_parent     46
		data->chip->irq_set_affinity
	irq_data_get_affinity_mask       47
	irq_set_vcpu_affinity    47
	irq_set_affinity_hint    167
		irq_set_affinity_and_hint
			__irq_apply_affinity_hint
				affinity_hint
				__irq_set_affinity ->
	struct irq_affinity     265

	15 irq_settings_is_per_cpu_devid
		_IRQ_PER_CPU_DEVID
	16 irq_cpu_rmap_add
		irq_set_affinity_notifier ->
	16 irq_set_percpu_devid
		irq_set_percpu_devid_partition
			irq_set_percpu_devid_flags
	17 irqd_is_forwarded_to_vcpu
		__irqd_to_state(d) & IRQD_FORWARDED_TO_VCPU;
	19 handle_percpu_devid_irq ->
	26 free_irq_cpu_rmap
		irq_set_affinity_notifier ->
	40 request_percpu_irq
		__request_percpu_irq ->
	33 free_percpu_irq
		__free_percpu_irq
	44 disable_percpu_irq
		irq_percpu_disable
	48 enable_percpu_irq
		irq_percpu_enable

	echo 4 > /proc/irq/24/effective_affinity_list

	irq_set_affinity
		args
			irq
			cpumask
		not forced
		__irq_set_affinity
			args
				irq
				cpumask
			raw_spin_lock_irqsave
			irq_set_affinity_locked
				args
					irq_data
					cpumask
				irq_set_affinity_deactivated
					cpumask_copy(desc->irq_common_data.affinity, mask);
					irq_data_update_effective_affinity ->
				irq_can_move_pcntxt
					irqd_can_move_in_process_context
						IRQD_MOVE_PCNTXT
							< irq_settings_can_move_pcntxt
				irq_try_set_affinity
					irq_do_set_affinity ->
					irq_set_affinity_pending
						irq_data_to_desc
						irqd_set_move_pending
							IRQD_SETAFFINITY_PENDING
						irq_copy_pending
							pending_mask
			raw_spin_unlock_irqrestore

	hisi_sas_v2_interrupt_preinit
		devm_platform_get_irqs_affinity
			platform_irq_count
			irq_calc_affinity_vectors
			irq_create_affinity_masks ->
			irq_update_affinity_desc
				irq_affinity_desc
				!irqd_affinity_is_managed
				affinity->is_managed
					IRQD_AFFINITY_MANAGED
					IRQD_MANAGED_SHUTDOWN
				cpumask_copy(desc->irq_common_data.affinity, &affinity->mask);


==== irq

	irq_setup_generic_chip
		irq_modify_status
			irqd_clear
				IRQD_NO_BALANCING | IRQD_PER_CPU | IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
			irq_settings_can_move_pcntxt
				_IRQ_MOVE_PCNTXT
			irq_settings_clr_and_set
			IRQD_MOVE_PCNTXT
			irqd_set

	CONFIG_GENERIC_IRQ_DEBUGFS
		x86_vector_domain_ops
			x86_vector_debug_show
	GENERIC_IRQ_DEBUGFS
			irq_debugfs_init
				irq_add_debugfs_entry
					dfs_irq_ops
						irq_debug_open
							irq_debug_show
								irqd_get
									__irqd_to_state
	irq_sysfs_init
		irq_lock_sparse
			mutex_lock
			sparse_irq_lock

	irq_lock_sparse ->
	NMI: Non-maskable interrupts
	LOC: Local timer interrupts
	SPU: Spurious interrupts
	PMI: Performance monitoring interrupts
	IWI: IRQ work interrupts
	RTR: APIC ICR read retries
	RES: Rescheduling interrupts
	CAL: Function call interrupts
	TLB: TLB shootdowns
	TRM: Thermal event interrupts
	THR: Threshold APIC interrupts
	DFR: Deferred Error APIC interrupts
	MCE: Machine check exceptions
	MCP: Machine check polls
	ERR:
	MIS:
	PIN: Posted-interrupt notification event
	NPI: Nested posted-interrupt event
	PIW: Posted-interrupt wakeup event

	__setup
		__setup_param
	tsc_setup
		tsc_clocksource_reliable

	send_call_function_ipi_mask

	"isolcpus="
		housekeeping_isolcpus_setup
			tick domain managed_irq

	DECLARE_DELAYED_WORK
	DECLARE_DEFERRABLE_WORK
	__queue_delayed_work
		housekeeping_any_cpu
		delayed_work_timer_fn
			__queue_work ->
		add_timer_global
		add_timer_on

	arch_timer_of_init
		arch_timer_register
			request_percpu_irq
			"arch_timer"
			arch_timer_evt
			arch_timer_handler_phys
				timer_handler
					evt->event_handler

	handle_percpu_devid_irq
		bpftrace -e 'kprobe:handle_percpu_devid_irq { @[sym(((struct irq_desc *)arg0)->action->handler)] = count();}'
			ipi_handler ->
			arch_timer_handler_phys
		trace_irq_handler_entry
			irq_handler_entry
				irq_handler_entry: irq=12 name=arch_timer
		res = action->handler(irq, action->dev_id);
		trace_irq_handler_exit
			"irq_handler_exit"


	trace_irq_handler_exit
		<-
		handle_fasteoi_nmi
		handle_percpu_devid_irq
		handle_percpu_devid_fasteoi_nmi
		__handle_irq_event_percpu


	trace_softirq_entry
	"softirq_entry"
	trace_softirq_exit
	"softirq_exit"

	trace_softirq_raise
		"softirq_raise"
		softirq_raise
		DEFINE_EVENT ->


	can_stop_full_tick
	check_tick_dependency

	tick_irq_exit
		tick_nohz_irq_exit
			tick_nohz_full_update_tick

				__tick_nohz_full_update_tick
					can_stop_full_tick
						check_tick_dependency
							TICK_DEP_MASK_POSIX_TIMER
							trace_tick_stop
							"tick_stop"
					tick_nohz_stop_sched_tick
					tick_nohz_restart_sched_tick

==== MSI

		msi_create_device_irq_domain
			msi_domain_for_each_desc
				msi_domain_first_desc
					msi_find_desc
						xa_for_each_start
				msi_next_desc
					msi_find_desc
			msi_setup_device_data
				msi_device_data
				msi_sysfs_create_group
					msi_irqs_group
						"msi_irqs"
				msi_device_data_release

			msi_domain_populate_irqs
				irq_set_msi_desc
					irq_set_msi_desc_off
						desc->irq_common_data.msi_desc = entry;
			msi_domain_depopulate_descs

		ice_alloc_irq
			pci_msix_can_alloc_dyn
				pci_msi_domain_supports
					MSI_FLAG_PCI_MSIX_ALLOC_DYN
			pci_msix_alloc_irq_at
				msi_domain_alloc_irq_at
						__msi_domain_alloc_irq_at
							__msi_domain_alloc_irqs
								__irq_domain_alloc_irqs
								msi_sysfs_populate_desc
									nvec_used
									sysfs_add_file_to_group
								irq_set_msi_desc_off ->

		mlx5_msix_alloc
		mlx5_irq_alloc
		mlx5_vdpa_dev_add
			allocate_irqs
				pci_msix_alloc_irq_at

		irq_get_msi_desc
			irq_get_irq_data
				irq_to_desc
				irq_data
			irq_data
			irq_common_data
			msi_desc

		irq_data_get_msi_desc
			irq_get_irq_data
			d->common->msi_desc;

==== irq_ipi

	IPI deferral
		text_poke_sync
			on_each_cpu ->
				on_each_cpu_cond_mask
			&do_sync_core
				sync_core
					serialize
		vunmap
			?
				vm_area_unmap_pages
					vunmap_range
						flush_tlb_kernel_range ->
							? kernel_tlb_flush_all
		trace_ipi_send_cpu
		trace_ipi_send_cpumask
		trace_csd_queue_cpu
		sched_switch
		ipi_send_cpumask
		sched_clock_stable
		trace_csd_function_entry
		trace_csd_function_exit
		filter by cpumask
			trace-cmd record -e 'sched_switch' -f "CPU & CPUS{$ISOLATED_CPUS}" \
				 -e 'sched_wakeup' -f "target_cpu & CPUS{$ISOLATED_CPUS}" \
				 -e 'ipi_send_cpu' -f "cpu & CPUS{$ISOLATED_CPUS}" \
				 -e 'ipi_send_cpumask' -f "cpumask & CPUS{$ISOLATED_CPUS}" \
				 hackbench
		ct_kernel_enter
			ct_kernel_enter_state kernel/context_tracking.c
		update_mds_branch_idle
			mds_idle_clear Microarchitectural Data Sampling vulnerabilities

		__sched_clock_stable
		mark_tsc_unstable
			clear_sched_clock_stable
				__clear_sched_clock_stable
					&__sched_clock_work
						__sched_clock_stable
			disable_sched_clock_irqtime
		CONFIG_VMAP_STACK
			arm, arm64
				handle_bad_stack
					panic_bad_stack
				init_irq_stacks
			handle_stack_overflow

	update_curr ->
		trace_sched_stat_runtime
		irq/338-ice-ens-4479    [009] d...2.. 94815.741415: sched_stat_runtime: comm=irq/338-ice-ens pid=4479 runtime=16111 [ns] vruntime=0 [ns]

		ipi_send_cpu
		smp_send_reschedule
			trace_ipi_send_cpu
				wakeup_preempt old: check_preempt_curr
			arch_smp_send_reschedule
				native_smp_send_reschedule
					__apic_send_IPI(cpu, RESCHEDULE_VECTOR);
						send_IPI
							default_send_IPI_single
								__apic_send_IPI_mask
									send_IPI_mask
										? flat_send_IPI_mask
											_flat_send_IPI_mask
												__default_send_IPI_dest_field
													native_apic_mem_write
	}
														alternative_io
	irq_work_raise

	{
		cpus_isolated_store
		cpus_timer_store
	}

	{
		irq_alloc_descs
			__irq_alloc_descs
		devm_irq_alloc_descs
			__devm_irq_alloc_descs
		__irq_alloc_descs
			irq_find_free_area
			irq_expand_nr_irqs
			mutex_lock(&sparse_irq_lock);
			alloc_descs
				alloc_desc
					kzalloc_node
						kmalloc_node
					init_desc
						alloc_masks
							zalloc_cpumask_var_node
							desc->irq_common_data.affinity
						desc_set_defaults
							desc_smp_init(desc, node, affinity);
								irq_default_affinity
								cpumask_copy(desc->irq_common_data.affinity, affinity);
				irq_insert_desc
					sparse_irqs
					mas_store_gfp
	}

	handle_edge_irq
	handle_percpu_irq
	irq_ack
		apic_ack_edge
			apic_ack_irq
				irq_move_irq
					irqd_is_setaffinity_pending
						IRQD_SETAFFINITY_PENDING
					__irq_move_irq
						irqd_irq_masked
							IRQD_IRQ_MASKED
						irq_move_masked_irq
							!irqd_is_per_cpu
							irq_do_set_affinity ->
							err: irqd_set_move_pending

	cpuset
		cpuset_cpus_allowed
		cpuset_flagbits_t

	irq_fixup_move_pending
		!irqd_is_setaffinity_pending
		pending_mask
		irqd_clr_move_pending
			~IRQD_SETAFFINITY_PENDING

	irq_force_complete_move
		irq_domain_get_irq_data
		apic_chip_data
		x86_vector_domain
		free_moved_vector
			irq_matrix_free
			vector_irq

	irq_shutdown_and_deactivate
		irq_shutdown ->
		irq_domain_deactivate_irq
			__irq_domain_deactivate_irq
				(x86_vector_domain_ops)
				x86_vector_deactivate
					vector_assign_managed_shutdown
						apic_update_irq_cfg ->
							irq_data_update_effective_affinity ->
			irqd_clr_activated

	start_kernel
	setup_arch
		.init.init_platform
		kvm_init_platform
			kvmclock_init
				cpuhp_setup_state
					__cpuhp_setup_state
						__cpuhp_setup_state_cpuslocked ->

	nt cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);

	HK_TYPE_MANAGED_IRQ

	kernel/locking/lockdep.c
	__lock_acquire
		mark_usage
			mark_lock
				mark_lock_irq
					valid_state
						print_usage_bug
							"WARNING: inconsistent lock state"
							"inconsistent {%s} -> {%s} usage"

	LOCKDEP_STATE
	 IN-HARDIRQ-W - holding a lock within an interrupt context with interrupts disabled
	 HARDIRQ-ON-W - attempting to acquire a lock with interrupts enabled

	lockdep
		print_deadlock_bug
			print_deadlock_scenario
		held_lock
			acquire_ip
		lock_acquire ->
	timerfd_tmrproc

	get_cpu_device

	device_add_attrs
		dev_attr_online
			online_store ->
			online_show
				device_lock(dev);
				!dev->offline;

	calls:
		entry_SYSCALL_64_after_hwframe
		do_syscall_64
		ksys_write
		vfs_write
		kernfs_fop_write_iter
			online_store
				lock_device_hotplug_sysfs
					if (mutex_trylock(&device_hotplug_lock))
				device_online
					device_lock
						mutex_lock(&dev->mutex);
					cpu_subsys_online ->
					device_unlock
				device_offline
					device_lock(dev);
					cpu_subsys_offline ->
				unlock_device_hotplug
					mutex_unlock
					device_hotplug_lock

	HK_TYPE_TIMER ->

	waiman longman
		cpuset-isolation
			sched/isolation: Enable runtime update of housekeeping cpumasks
				+housekeeping_exclude_cpumask
				+cpumask_lock
			sched/isolation: Call sched_tick_offload_init() if housekeeping_overridden is ever set
				+sched_tick_offload_inited
				+sched_tick_offload_init
			sched/isolation: Use RCU to delay successive housekeeping cpumask updates
				synchronize_rcu
				call_rcu(&rcu_gp, rcu_gp_end);
			cgroup: Add a new CSS_KILLED flag
			cgroup/cpuset.c: Fix a partition error with CPU hotplug
				*update_parent_effective_cpumask
				*cpuset_hotplug_update_tasks
			cgroup/cpuset: Remove the unnecessary css_get/put() in cpuset_partition_write()
			cgroup/cpuset: Exclude isolated CPUs from housekeeping CPU masks
				HOUSEKEEPING_FLAGS
				update_isolation_cpumasks <- update_unbound_workqueue_cpumask
					+housekeeping_exclude_cpumask
			cgroup/cpuset: Use CPU hotplug code to enable proper transition of isolated CPUs
				*isolcpus_update_mutex
				+isolcpus_update_state
				*update_isolation_cpumasks
					+remove_cpu
					+cpus_read_lock/cpus_read_unlock
					+add_cpu
				...
					+mutex_lock(&isolcpus_update_mutex)
					+update_isolation_cpumasks
					+mutex_unlock(&isolcpus_update_mutex);
			cgroup/cpuset: Don't call rebuild_sched_domains*() unnecessarily
			sched/core: Don't check DL BW in sched_cpu_deactivate() if CPU is isolating
				cpuset_cpu_is_isolating
				*sched_cpu_deactivate


		pool_workqueue
			+plugged
		unplug_oldest_pwq
		init_rescuer
			+wq_unbound_cpumask
		ordered_workqueue_ref_check
		workqueue_apply_unbound_cpumask
			ordered_workqueue_ref_check
		apply_wqattrs_commit
			+unbound_effective_cpumask
				unbound_pwq
					unbound_pwq_slot
				__pod_cpumask
		apply_wqattrs_prepare
			__WQ_ORDERED plugged

		apply_wqattrs_cleanup
		pwq_tryinc_nr_active
			pool_workqueue
				frozen
		pwq_release_workfn
			+thaw_pwq
		rcu_nocb_cpumask_update frederic
			rcu_nocb_cpu_deoffload ->
		rcu_nocb_enabled
		update_isolation_cpumasks w2 <- update_unbound_workqueue_cpumask
			lockdep_assert_cpus_held ->
			workqueue_unbound_exclude_cpumask ->
			HOUSEKEEPING_FLAGS
			housekeeping_exlude_isolcpus w1
				boot_hk_cpumask
				boot_hk_flags
				lockdep_assert_cpus_held ->
				+ my housekeeping_update ->
				WRITE_ONCE
					__WRITE_ONCE
						volatile
				housekeeping.flags
			rcu_nocb_mask_preset
		cpuset.cpus.isolation_full, isolation_full

		wq_requested_unbound_cpumask
		hk=wq_isolated_cpumask
		workqueue_unbound_exclude_cpumask ->
		dfl_files
			+cpuset_write_isolfull
				pdate_isolation_cpumasks

	irq_affinity_online_cpu
		irq_lock_sparse ->
		irq_restore_affinity_of_irq ->
			irqd_affinity_is_managed
			irqd_is_managed_and_shutdown
				IRQD_MANAGED_SHUTDOWN
			irq_startup ->
			irqd_is_single_target
				IRQD_SINGLE_TARGET
			hk_should_isolate ->
				HK_TYPE_MANAGED_IRQ
			irq_set_affinity_locked ->
				... irq_do_set_affinity->
					... apic_update_irq_cfg ->

	housekeeping
		housekeeping_setup ->
			housekeeping_setup_type ->
		housekeeping_init < start_kernel
			static_branch_enable(&housekeeping_overridden);
		housekeeping_enabled
			!!(housekeeping.flags & BIT(type));
		housekeeping_cpumask (const)
			housekeeping.flags & BIT(type)
			housekeeping.cpumasks[type];
			cpu_possible_mask; 
				SC_NPROCESSORS_CONF  possible
		housekeeping_cpu
			housekeeping_test_cpu(cpu, type);
				if (housekeeping.flags & BIT(type))
					return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
		housekeeping_any_cpu
			sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
				sched_domains_numa_masks
	HK_TYPE_TIMER used by:
		__queue_delayed_work
			add_timer_on(timer, cpu);
		try_to_generate_entropy
			< wait_for_random_bytes urandom_read_iter
			cpumask_and(&timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask);
			add_timer_on
				get_timer_cpu_base ->
				forward_timer_base
				internal_add_timer ->
			add_timer_on(&stack->timer, cpu);
		lockup_detector_init
			< kernel_init_freeable ->
			cpumask_copy(&watchdog_cpumask,
		     		housekeeping_cpumask(HK_TYPE_TIMER));

			...
				lockup_detector_reconfigure
					__lockup_detector_reconfigure ->
		freeze_secondary_cpus, /CONFIG_PM_SLEEP_SMP/
		< hibernate_resume_nonboot_cpu_disable hibernate_resume_nonboot_cpu_disable hibernate_resume_nonboot_cpu_disable suspend_disable_secondary_cpus
			_cpu_down
				cpuhp_tasks_frozen
				cpuhp_down_callbacks
				cpuhp_invoke_callback
					takedown_cpu CPUHP_TEARDOWN_CPU
						cpuhp_state
						kthread_park
						stop_machine_cpuslocked ->
						&take_cpu_down
							CPUHP_AP_OFFLINE
							cpuhp_state
							__cpu_disable ->
								native_cpu_disable ->
							cpuhp_invoke_callback_range_nofail
								__cpuhp_invoke_callback_range
									cpuhp_invoke_callback ->
							...
								tick_cpu_dying
								hrtimers_cpu_dying
								smpcfd_dying_cpu
								x86_pmu_dying_cpu
								rcutree_dying_cpu
								sched_cpu_dying
								cache_ap_offline
							native_cpu_disable
							stop_machine_park
								cpu_stopper
								kthread_park
						arch_cpuhp_cleanup_dead_cpu
						tick_cleanup_dead_cpu
						rcutree_migrate_callbacks
		get_nohz_timer_target
			<
				switch_hrtimer_base > get_target_base
				__mod_timer > get_target_base
			idle_cpu
				sudo bpftrace  -e 'kprobe:idle_cpu {@cpu[tid] = arg0;} kretprobe:idle_cpu {printf("%u %u\n", @cpu[tid], retval);delete(@cpu[tid]);} END{clear(@cpu);}'
			housekeeping_cpu(cpu, HK_TYPE_TIMER))
			migrate_hrtimer_list ->
			CPUHP_TIMERS_PREPARE
				timers_prepare_cpu
					per_cpu_ptr(&timer_bases[b], cpu);
					is_idle = false
				timers_dead_cpu
					get_cpu_ptr ->
					raw_spin_lock_irq
						_raw_spin_lock_irq
							__raw_spin_lock_irq
                                                                local_irq_disable
                                                                preempt_disable
                                                                do_raw_spin_lock ->
					raw_spin_lock_nested
					timer_bases
					migrate_timer_list
						hlist_entry
						detach_timer
							debug_deactivate
							__hlist_del
						internal_add_timer
					put_cpu_ptr ->
			housekeeping_any_cpu(HK_TYPE_TIMER);
		kvm_x86_vendor_init
			pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
			kvm_can_post_timer_interrupt
				pi_inject_timer
	HK_TYPE_MANAGED_IRQ is used by:
		... irq_do_set_affinity ->
			< irq_setup_affinity ->
			chip->irq_set_affinity
		housekeeping_isolcpus_setup
			"domain" HK_FLAG_DOMAIN
			"managed_irq" HK_FLAG_MANAGED_IRQ
			housekeeping_setup ->
		cpuhp_hp_states
		- irq_affinity_online_cpu
			irq_restore_affinity_of_irq
				irq_data_get_affinity_mask ->
				hk_should_isolate
					< irq_restore_affinity_of_irq ->
					irq_data_get_effective_affinity_mask
					hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
					cpumask_test_cpu
				irq_set_affinity_locked ->
					... irq_do_set_affinity->
						housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
						irq_common_data.affinity


	HK_TYPE_KTHREAD is used by:
		unbound - NUMA_NO_NODE
	create_kthread
		kthreadd ->
			set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD));
		sysctl_est_cpulist
			housekeeping_cpumask(HK_TYPE_KTHREAD);
		ipvs_proc_est_cpumask_get
			mask = (struct cpumask *)housekeeping_cpumask(HK_TYPE_KTHREAD);
	housekeeping_cpumask
		used by:
		task_cpu_fallback_mask
			housekeeping_cpumask(HK_TYPE_TICK)
		print_cpus_isolated
		try_to_generate_entropy
		drivers/hv/
			init_vp_index
			vmbus_channel_set_cpu
		pci_call_probe
		sysctl_est_cpulist
		sysctl_est_cpulist
		generate_sched_domains ->
		cpuset_init
		cpu_down_maps_locked
		hk_should_isolate
		irq_do_set_affinity
		kthread_fetch_affinity
		kthreadd
		swake_up_one_online
		get_nohz_timer_target
		sched_init_smp
		scx_ops_enable
		find_new_ilb
		asym_cpu_capacity_scan
		sched_init_domains
		partition_sched_domains_locked ->
		get_target_base
		hrtimers_cpu_dying
		lockup_detector_init
		workqueue_init_early
		rps_cpumask_housekeeping
		ipvs_proc_est_cpumask_get

	HK_TYPE_KERNEL_NOISE
		used by:
		get_nohz_timer_target
		sched_tick
		sched_tick_start
		sched_tick_stop
		find_new_ilb

		HK_TYPE_RCU
			rcu_boost_kthread_setaffinity
			rcu_bind_gp_kthread
			rcu_tasks_kthread
		HK_TYPE_MISC
			create_core_data
			find_new_ilb
				nohz.idle_cpus_mask
				ILB - Idle Load Balancing
		HK_TYPE_TICK
			cpu_is_isolated ->
				used by:
				blk_mq_map_swqueue
				drain_all_stock
				vmstat_shepherd
				fs/buffer.c:
					bh_lru_install
					lookup_bh_lru

			sched_tick <- scheduler_tick
			sched_tick_start
			sched_tick_stop
				TICK_SCHED_REMOTE_OFFLINING
	x HK_TYPE_SCHED
		nohz_balance_enter_idle
		nohz_newidle_balance
	HK_TYPE_DOMAIN,
		print_cpus_offline
			grep '' /sys/devices/system/cpu/{offline,online,possible,present}
			cpumask_andnot(offline, cpu_possible_mask, cpu_online_mask);
		print_cpus_isolated
			cpu_possible_mask & !housekeeping_cpumask(HK_TYPE_DOMAIN)
		pci_call_probe
		cpu_is_isolated ->
		generate_sched_domains ->
		prstate_housekeeping_conflict
			boot_hk_cpus
		cpu_down_maps_locked
		sched_init_smp
			cpu_active_mask
				< set_cpu_active
			sched_init_domains
				asym_cpu_capacity_scan
				alloc_sched_domains
			set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN))
		partition_sched_domains_locked ->
		rps_cpumask_housekeeping
	HK_TYPE_WQ
		Q: /sys/devices/virtual/workqueue/cpumask
			workqueue_sysfs_register
		pci_call_probe
		rps_cpumask_housekeeping

		workqueue_init_early
			restrict_unbound_cpumask
		update_cpumask
			rcu_read_lock
			cpuset_for_each_child
				css_for_each_child
					css_next_child
						list_entry_rcu
						list_for_each_entry_rcu
			remote_cpus_update
				spin_lock_irq(&callback_lock);
				partition_xcpus_add
					partition_xcpus_newstate
						isolated_cpus
				partition_xcpus_del
					cpumask_and(xcpus, xcpus, cpu_active_mask);
					partition_xcpus_newstate ->
				update_isolation_cpumasks ->
				top_cpuset
				cpuset_update_tasks_cpumask
					css_task_iter_start
					set_cpus_allowed_ptr
				update_sibling_cpumasks
					rcu_read_lock
					update_cpumasks_hier
						cpuset_for_each_descendant_pre
		cpuset_css_offline > update_prstate
		"cpus.partition"
		- sched_partition_show
		- - partition_root_state
		- sched_partition_write
			is_cpuset_online
		- - "isolated"
		- - cpus_read_lock
		- - mutex_lock(&cpuset_mutex);
		- - update_prstate
			update_partition_exclusive
				CS_CPU_EXCLUSIVE
			partition_root_state
			remote_partition_enable remote_partition_disable
				spin_lock_irq(&callback_lock);
				partition_xcpus_add ->
				update_unbound_workqueue_cpumask ->
				top_cpuset
				update_tasks_cpumask
					css_task_iter_next
					subpartitions_cpus
					set_cpus_allowed_ptr
				update_sibling_cpumasks
					update_cpumasks_hier
			update_parent_effective_cpumask ->
			update_unbound_workqueue_cpumask ->
			spin_lock_irq(&callback_lock);
			partition_xcpus_newstate ->
			update_isolation_cpumasks ->
			rebuild_sched_domains_locked
		wq_unbound_cpumask_store
			workqueue_set_unbound_cpumask
				wq_requested_unbound_cpumask
				workqueue_apply_unbound_cpumask ->

		cpuhp_hp_states
			workqueue_online_cpu
				wq_update_pod
				rebind_workers
					kthread_set_per_cpu
				unbound_wq_update_pwq ->
			workqueue_offline_cpu
				unbind_workers
					unbind_worker
						kthread_set_per_cpu -1
				wq_update_pod
					wq_calc_pod_cpumask

	arm
	PMU Performance Monitoring Unit
	runc
		rmqueue

		__arm64_sys_prctl
		__do_sys_prctl
			sys_prctl
				PR_SET_SECCOMP
				prctl_set_seccomp
					do_seccomp
						seccomp_set_mode_strict not called
							spin_lock_irq
							current->sighand->siglock
							seccomp_assign_mode
								arch_seccomp_spec_mitigate
						seccomp_set_mode_filter 4683 us
							...
							seccomp_may_assign_mode
							has_duplicate_listener
							spin_lock_irq
							seccomp_attach_filter 275 us
								seccomp_can_sync_threads
									for_each_thread
								seccomp_cache_prepare ?
									seccomp_cache_prepare_bitmap
										for (nr = 0; nr < bitmap_size; nr++) {
										seccomp_is_const_allow
								seccomp_sync_threads
									for_each_thread
										list_for_each_entry_rcu ->
							seccomp_assign_mode ..4 us
						seccomp_get_action_avail
						seccomp_get_notif_sizes
						_raw_spin_unlock_irq
							el1h_64_irq
								...
									hrtimer_wakeup
	 seccomp
	 	do_seccomp
			 seccomp_set_mode_filter
			 bpf_prog_create_from_user
			 bpf_prepare_filter
			 bpf_prog_select_runtime
			 bpf_int_jit_compile
			 bpf_jit_binary_pack_alloc
			 bpf_prog_pack_alloc
			 alloc_new_pack
			 set_memory_ro
			 change_page_attr_set_clr
			 _vm_unmap_aliases.part.0
			 __purge_vmap_area_lazy
			 	flush_tlb_kernel_range ->
					on_each_cpu ->
					&do_flush_tlb_all
						__flush_tlb_all
							__flush_tlb_global
								native_flush_tlb_global
							flush_tlb_local
								__flush_tlb_local
									native_flush_tlb_local
					get_flush_tlb_info
					on_each_cpu
					do_kernel_range_flush
						flush_tlb_one_kernel
					put_flush_tlb_info
			 on_each_cpu_cond_mask
			 	smp_call_function_many_cond
			 ipi_send_cpumask
			 trace_event_raw_event_ipi_send_cpumask


==== Softirqs

	show_softirqs
		NR_SOFTIRQS
		kstat_softirqs_cpu

	see rcutree.use_softirq=


==== NMI

	request_nmi
		__setup_irq
		irq_nmi_setup

	arch/x86/include/asm/nmi.h

	arch/x86/kernel/nmi.c


	in_nmi
		nmi_count
			NMI_MASK

	include/linux/hardirq.h

	arch/x86/kernel/nmi.c
		register_nmi_handler
			__register_nmi_handler
		exc_nmi
			irqentry_nmi_enter
				__nmi_enter
				default_do_nmi
					nmi_handle
						nmiaction
						handler
						trace_nmi_handler - nmi_handler
						nmi_check_duration
			irqentry_nmi_exit

init_hw_perf_events
nmi_handler: perf_event_nmi_handler() delta_ns: 8750 handled: 1
	perf_sample_event_took

register_nmi_cpu_backtrace_handler
nmi_handler: nmi_cpu_backtrace_handler() delta_ns: 569 handled: 0
	nmi_cpu_backtrace

ghes_nmi_add
nmi_handler: ghes_notify_nmi() delta_ns: 8745 handled: 0
	atomic_add_unless


	arch/x86/kernel/traps.c
		exc_double_fault
		exc_int3
		exc_debug
			exc_debug_kernel


	nmi_enter
		__nmi_enter
			arch_nmi_enter
			__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET)

	fixup_ht_bug
		hardlockup_detector_perf_restart

=== Preemption

	preempt_count_add
		__preempt_count_add



==== dynticks (nohz)

CONFIG_NO_HZ_FULL
	task,signal_struct::tick_dep_mask

	tick_nohz_init ->
	tick_nohz_handler -> (9.4 tick_sched_timer, 9.5+ tick_nohz_highres_handler)

	tick_nohz_full_running

	sched_tick_start
		tick_work
		TICK_SCHED_REMOTE_RUNNING
		& sched_tick_remote
			curr->sched_class->task_tick
				task_tick_fair ->
				task_tick_rt ->

	other
		account_freq_event_nohz
		unaccount_freq_event_nohz
		__rcu_irq_enter_check_tick
		sched_can_stop_tick
		sched_tick_offload_init

	referrers tick_nohz_full_cpu
		mon_event_read
		cpumask_any_housekeeping
		tick_dep_set_cpu
		tick_dep_clear_cpu
		tick_nohz_user_enter_prepare
		check_holdout_task
		show_stalled_task_trace
		__rcu_irq_enter_check_tick
		rcu_disable_urgency_upon_qs
		rcu_watching_snap_recheck
		rcu_nohz_full_cpu
		wake_up_full_nohz_cpu
		sched_fair_update_stop_tick
		sched_update_tick_dependency
		irq_enter_rcu
		tick_irq_exit
		tick_setup_device
		tick_nohz_full_kick
		tick_nohz_full_kick_cpu
		__tick_nohz_task_switch
		tick_nohz_full_update_tick
		tick_nohz_idle_update_tick
		trigger_dyntick_cpu
		__get_next_timer_interrupt
		timer_clear_idle

	/sys/devices/system/cpu/nohz_full
		print_cpus_nohz_full
			< cpu_root_attrs
			tick_nohz_full_mask

	unused
		tick_nohz_full_add_cpus_to
			cpumask_or

	tick_nohz_dep_set
		tick_nohz_dep_set_all
			tick_nohz_full_kick_all
				tick_nohz_full_mask
				tick_nohz_full_kick_cpu ->
		

	dynticks
		wake_up_nohz_cpu
			wake_up_full_nohz_cpu
				tick_nohz_full_cpu
					tick_nohz_full_enabled
						context_tracking_enabled
							context_tracking_key
						tick_nohz_full_running
					tick_nohz_full_mask
			tick_nohz_full_kick_cpu
				irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);

	CONFIG_NO_HZ_FULL NO_HZ_FULL
	start_kernel
		tick_init
			tick_broadcast_init
			tick_nohz_init
				tick_nohz_full_mask
				context_tracking_cpu_set
				cpuhp_setup_state_nocalls
				CPUHP_AP_ONLINE_DYN,
				&tick_nohz_cpu_down
					tick_nohz_cpu_hotpluggable
	x
		rcu_user_enter
		rcu_user_exit
	rcu_nohz_full_cpu
	sched_can_stop_tick
	housekeeping_init
		sched_tick_offload_init
			tick_work_cpu

	__setup
	"nohz_full" sets housekeeping to negative argument
	housekeeping_nohz_full_setup
		HK_FLAG_TICK HK_FLAG_WQ HK_FLAG_TIMER HK_FLAG_RCU HK_FLAG_MISC HK_FLAG_KTHREAD
		sets housekeeping.cpumasks for tick, wq, timer, rcu, misc, and kthread
		housekeeping_setup sets housekeeping to negative argument
			alloc_bootmem_cpumask_var
				memblock_alloc
					memblock_alloc_try_nid
						memblock_alloc_internal
							memblock_alloc_range_nid
								memblock_find_in_range_node
									__memblock_find_range_bottom_up
										for_each_free_mem_range
											memblock
			cpu_present_mask
			housekeeping_setup_type(type, housekeeping_staging);
				cpumask_copy(housekeeping.cpumasks[type], housekeeping_staging);
			tick_nohz_full_setup
				cpumask_copy
				tick_nohz_full_mask
				tick_nohz_full_running = true;
	show_state_filter
		sched_show_task
		sysrq_sched_debug_show kernel/sched/debug.c
			"Sched Debug Version"
			print_cpu

	4 apic_timer_interrupt➝
		smp_apic_timer_interrupt➝
			hrtimer_interrupt CONFIG_HIGH_RES_TIMERS
				max_hang_time 
					grep -E 'cpu:|max_hang_time' /proc/timer_list
				now = hrtimer_update_base(cpu_base);
				ktime_before
				__hrtimer_run_queues➝
				raise_hrtimer_softirq
					HRTIMER_SOFTIRQ
					raise_ktimers_thread
						__this_cpu_or(pending_timer_softirq, 1 << nr);
				tick_program_event


	static inline void hrtimer_set_expires_range
		timer->_softexpires = time;
		timer->node.expires = ktime_add_safe(time, delta);

	hrtimer_get_expires
			timer->node.expires

		hrtimer_get_softexpires
			timer->_softexpires

	__ro_after_init
		apic_timer_interrupt

	asm_sysvec_apic_timer_interrupt➝sysvec_apic_timer_interrupt➝__sysvec_apic_timer_interrupt➝

	sysvec_apic_timer_interrupt
		local_apic_timer_interrupt
			event_handler
				-> tick_handle_periodic
					tick_periodic
						do_timer ->
							jiffies_64 ++
							jiffies++;
								x update_times
									update_wall_time
										timekeeping_advance
											timekeeping_adjust
											timekeeping_update
										clock_was_set_delayed
							calc_global_load
								calc_load_tasks
								avenrun
								calc_load
						update_process_times
							run_local_timers
								hrtimer_run_queues ->
							sched_tick
								arch_scale_freq_tick
									rdmsrq
										native_read_msr ->

	hrtimer_run_queues
		tick_switch_to_oneshot

		raw_spin_lock_irqsave(&cpu_base->lock, flags); ->
		tick_check_oneshot_change
			tick_nohz_lowres_handler
				tick_nohz_handler
				tick_program_event
			tick_nohz_switch_to_nohz
				sched_timer
				hrtimer_init
				tick_init_jiffy_update
					jiffies_seq
					write_seqcount_begin
						seqprop_ptr
							__seqprop
								__seqprop_case
							seqcount
						do_write_seqcount_begin
							do_write_seqcount_begin_nested
								do_raw_write_seqcount_begin
									sequence++
								seqcount_acquire
					last_jiffies_update
					write_seqcount_end
						do_write_seqcount_end
							do_raw_write_seqcount_end
								sequence++
				tick_setup_sched_timer ->
		!hrtimer_is_hres_enabled
		hrtimer_switch_to_hres
			tick_init_highres
				tick_switch_to_oneshot(hrtimer_interrupt);
			tick_setup_sched_timer
				hrtimer_setup ->
				sched_timer
				&tick_nohz_handler -> (9.4 tick_sched_timer, 9.5+ tick_nohz_highres_handler)
				sched_skew_tick
				tick_program_event
				tick_nohz_activate
					tick_sched_flag_set(ts, TS_FLAG_NOHZ);
					timers_update_nohz
						schedule_work(&timer_update_work);


		...
		9.4:
			tick_nohz_switch_to_nohz
				tick_switch_to_oneshot
				tick_nohz_handler
		tick_nohz_handler
			sched_timer
			tick_sched_do_timer
				tick_do_update_jiffies64
				ts->last_tick_jiffies = READ_ONCE(jiffies);
			tick_sched_handle
				update_process_times
					run_local_timers
						hrtimer_run_queues ->
							hrtimer_switch_to_hres ->
								tick_setup_sched_timer
									&tick_init_jiffy_update
							__hrtimer_run_queues ->
					rcu_sched_clock_irq ->
					5 raise_timer_softirq
			hrtimer_forward ->

	hrtimer_run_softirq hrtimer_interrupt hrtimer_run_queues
	> __hrtimer_run_queues
		__run_hrtimer
			timer->function

		hrtimer_get_softexpires (hrtimer_get_softexpires_tv64)

		local_pending_timers
			__this_cpu_read(pending_timer_softirq)
	tick_sched_timer➝ tick_nohz_handler
	tick_sched_handle➝
	update_process_times➝
	tick_sched_do_timer➝
	scheduler_tick➝
	task_tick
		-> task_tick_fair
			entity_tick
				update_curr ->
				update_load_avg ->
				update_cfs_group ->
				resched_curr_lazy
					resched_curr
						set_tsk_need_resched ->
						set_preempt_need_resched
							PREEMPT_NEED_RESCHED
						smp_send_reschedule ->
					set_tsk_need_resched_lazy
						set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
				check_preempt_tick
			update_misfit_status
			update_overutilized_status

		spawn_ksoftirqd
			timer_thread "ktimers/%u"
				timersd
				6 run_ktimerd
				run_timerd ->
				5 run_timersd ->
					local_pending_timers
			smpboot_register_percpu_thread
				hotplug_threads
				smp_hotplug_thread
				__smpboot_create_thread
					smp_hotplug_thread.store
					kthread_create_on_cpu
					kthread_set_per_cpu
					&smpboot_thread_fn ->

	test_preempt_need_resched
		PREEMPT_NEED_RESCHED

	spawn_ksoftirqd
		softirq_threads "ksoftirqd/%u"
			run_ksoftirqd NRT
				__do_softirq ->

	ret_from_fork
		kthread
			smpboot_thread_fn
				kthread
				kthread_should_stop
				kthread_should_park
					__kthread_should_park

				6 run_ktimerd
				run_timerd RT
					__do_softirq
						hrtimer_run_softirq ->
						__hrtimer_run_queues
							tick_nohz_handler ->
							timerqueue_getnext
							__run_hrtimer
								-> sched_cfs_period_timer
				kthread_parkme
					__kthread_parkme
						set_special_state(TASK_PARKED);
				smp_hotplug_thread.thread_fn
	 < __run_hrtimer
	? set_kthread_struct

	5 sched_debug
	5 /sys/kernel/debug/sched/debug

	4 init_sched_debug_procfs
		"sched_debug" /proc/sched_debug
		sched_debug_sops
			sched_debug_show
				print_cpu kernel/sched/debug.c
					"cpu#"
					print_cfs_stats
						for_each_leaf_cfs_rq_safe
						cfs_rq
						print_cfs_rq
							cpu_rq
								per_cpu
									per_cpu_ptr
							"cfs_rq"
								task_group_path
									autogroup_path
									cgroup_path
										kernfs_path
											kernfs_path_from_node
							throttled
							throttle_count
							cfs_rq->throttle_count
							print_cfs_group_stats /proc/sched_debug
								schedstat_enabled
								sleep_max in ms
								block_max
								wait_max
								sum_exec_runtime
								vruntime

					print_rq
						"runnable tasks"
						print_task
							se.statistics.wait_sum
					print_rt_stats
						print_rt_rq
							"rt_rq"

	sched_schedstats

	echo 0 > events/workqueue/workqueue_execute_start/enable
	echo 0 > events/workqueue/workqueue_execute_end/enable
	echo 0 > events/workqueue/workqueue_activate_work/enable
	echo 0 > events/workqueue/workqueue_queue_work/enable

	echo 1 > events/sched/sched_stat_wait/enable
	echo 1 > events/sched/sched_stat_iowait/enable
	echo 1 > events/sched/sched_stat_blocked/enable

	echo 1 > events/sched/sched_stat_sleep/enable

	echo 1 > /proc/sys/kernel/sched_schedstats
	sysctl kernel.sched_schedstats=1
			set_schedstats

	echo 1 > /proc/sys/kernel/sched_schedstats
	cat /proc/self/sched

	grep se.statistics.wait_sum /proc/*/sched

	latency_warn_ms
	sysctl_resched_latency_warn_ms
		cpu_resched_latency
		scheduler_tick
			resched_latency_warn

	fair_sched_class
		set_next_task_fair
			set_next_entity
				update_stats_wait_end ->
				__dequeue_entity
					rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline)
				?	se->exec_start = rq_clock_task(rq_of(cfs_rq))u
				update_load_avg
					cfs_rq_clock_pelt
						rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time
						rq_clock_pelt
							rq->clock_pelt - rq->lost_idle_time
						cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time
				update_stats_curr_start
					se->exec_start = rq_clock_task(rq_of(cfs_rq));

	Async unthrottling for cfs bandwidth
		distribute_cfs_runtime
			unthrottle_cfs_rq_async
				__unthrottle_cfs_rq_async
					unthrottle_cfs_rq ->
			throttled_csd_list

		cfsb_csd
			__cfsb_csd_unthrottle
		cfsb_csd_list

	CONFIG_FAIR_GROUP_SCHED
		.task_change_group      = task_change_group_fair
	RT_GROUP_SCHED
	CGROUP_SCHED
		FAIR_GROUP_SCHED
			CONFIG_CFS_BANDWIDTH
				cfs_bandwidth_used
				sysctl_sched_cfs_bandwidth_slice
				tg_get_cfs_quota
				tg_set_cfs_quota ->
					tg_set_cfs_bandwidth ->
				cfs_rq_clock_pelt Per-Entity Load Tracking
				cfs_bandwidth_used ...
				cfs_bandwidth_usage_inc
					__cfs_bandwidth_used
				cfs_bandwidth_usage_dec
				cfs_rq +=
					runtime_enabled
					throttled_clock

					throttled_clock_task_time
					throttle_count
					...
	pick_next_task __pick_next_task_fair
		pick_next_task_fair kernel/sched/fair.c
			check_cfs_rq_runtime ^
			put_prev_entity ^
			set_next_entity
			hrtick_enabled_fair
				hrtick_enabled ->
			hrtick_start_fair ->
			+ hrtick_enabled_bw
				: sched_feat(HRTICK_BW)) && hrtick_enabled
				hrtick_enabled
					cpu_active(cpu_of(rq)))
					hrtimer_is_hres_active(&rq->hrtick_timer)

			+ start_hrtick_cfs_bw
				: tick_nohz_full_cpu && cfs_bandwidth_used && cfs_rq->runtime_enabled
				rq->nr_running == 1 && cfs_rq->runtime_remaining
				hrtick_start ->
			sched_balance_newidle
				sched_balance_rq
					&active_load_balance_cpu_stop
					stop_one_cpu_nowait
					cpu_stop_queue_work ->
	__pick_next_task_fair
		pick_next_task_fair ^

	sched_fork
		rt_sched_class

	enqueue_entity < ...
		4 update_stats_enqueue
			update_stats_wait_start
				wait_start = rq_clock(rq_of(cfs_rq));
					rq->clock
				trace_sched_stat_wait
				trace sched_stat_wait
			4 update_stats_enqueue_sleeper
		update_stats_enqueue_fair
			update_stats_wait_start_fair
				__update_stats_wait_start
			update_stats_enqueue_sleeper_fair
				schedstat_enabled
				__update_stats_enqueue_sleeper
						statistics.sleep_max
					trace_sched_stat_sleep
					trace_sched_stat_iowait
					trace_sched_stat_blocked

		__enqueue_entity
			__entity_less
			rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less);
		se->on_rq = 1
		check_enqueue_throttle
			account_cfs_rq_runtime ->
			throttle_cfs_rq ->
		update_cfs_group
			reweight_entity
				update_curr ->

	< throttle_cfs_rq dequeue_task_fair < fair_sched_class
	dequeue_entity
	return_cfs_rq_runtime
	__return_cfs_rq_runtime
		slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime
		cfs_rq->runtime_remaining -= slack_runtime

	reweight_task
		update_cfs_group

	rt_sched_class
		fair_sched_class : sched_class
			.enqueue_task
			enqueue_task_fair
				cfs_rq_throttled
				enqueue_entity ->
			put_prev_task_fair
				put_prev_entity
					check_cfs_rq_runtime
						throttle_cfs_rq ->
					update_stats_wait_start
					__enqueue_entity ->
					update_load_avg
				list_add_leaf_cfs_rq
					leaf_cfs_rq_list
				hrtick_update
					hrtick_start_fair
						sched_slice
							__sched_period
							update_load_add
							__calc_delta
						hrtick_start kernel/sched/core.c
							hrtimer
							hrtick_timer
							hrtick_time
							__hrtick_restart
								hrtick_timer
								hrtimer_start ->
								.. hrtick
							smp_call_function_single_async ->
							hrtick_csd

		.rq_offline < set_rq_offline
			rq_offline_fair
					unthrottle_offline_cfs_rqs
						list_for_each_entry_rcu(tg, &task_groups
								throttle_count
						cfs_rq->runtime_remaining = 1
						cfs_rq->runtime_enabled = 0
						unthrottle_cfs_rq ->

		throttle_cfs_rq
			__assign_cfs_rq_runtime
			list_add_tail_rcu(&cfs_rq->throttled_list,
					  &cfs_b->throttled_cfs_rq);
			cfs_rq->runtime_remaining <= 0 :
			walk_tg_tree_from
			tg_throttle_down
				list_add_tail_rcu(&cfs_rq->throttled_list,
						  &cfs_b->throttled_cfs_rq);
				throttled_clock_task = rq_clock_task
				throttle_count++
			dequeue_entity DEQUEUE_SLEEP
				update_curr ->
				update_load_avg
				4 update_stats_dequeue
				5 update_stats_wait_end_fair
					schedstat_enabled
						sched_schedstats
					__update_stats_wait_end
						delta
						wait_max
							grep -E 'wait_max *: [0-9]{3}' /sys/kernel/debug/sched/debug
						wait_coun
						wait_sum
						wait_start = 0
					sleep_start =
					block_start =
					__dequeue_entity ->
					se->on_rq = 0;
				account_entity_dequeue
				return_cfs_rq_runtime
				update_cfs_group
			cfs_rq->throttled = 1
			throttled_clock

	__cfsb_csd_unthrottle <..
		unthrottle_cfs_rq
			cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
			cfs_rq->throttled = 0
			list_del_rcu(&cfs_rq->throttled_list);
			tg_unthrottle_up
				throttle_count--
				throttled_clock_task_time += rq_clock_task - throttled_clock_task
				list_add_leaf_cfs_rq
			enqueue_entity ENQUEUE_WAKEUP ->
			rq_clock(rq) - cfs_rq->throttled_clock
			cfs_rq_throttled
				cfs_bandwidth_used
					__cfs_bandwidth_used
				cfs_rq->throttled
			list_add_leaf_cfs_rq ->
			resched_curr

	brew download-task
	child_cfs_rq_on_list

	cgcreate cgclassify

	do_syscall_64
		syscall_trace_enter
			trace_sys_enter
		syscall_exit_work
			trace_sys_exit

	SYSCALL_METADATA
		SYSCALL_TRACE_ENTER_EVENT
			print_syscall_enter
				"%s: "
		SYSCALL_TRACE_EXIT_EVENT
			print_syscall_exit
				" -> "

	entry_SYSCALL_64_after_hwframe➝do_syscall_64➝

	do_mkdirat➝vfs_mkdir➝kernfs_iop_mkdir➝
		cgroup_mkdir➝
			cgroup_apply_control_enable➝
				cpu_cgroup_css_alloc➝
					sched_create_group➝
				online_css
					cpuset_css_online
						cpus_read_lock
          					mutex_lock


	context_tracking_user_enter
		user_enter
			context_tracking_enter
				__context_tracking_enter
					trace_user_enter

	--skip-chdir --userspec=USER:GROUP

	retint_user➝
	prepare_exit_to_usermode➝
	exit_to_usermode_loop➝
	schedule➝
	trace_hardirqs_off_thunk➝
	__schedule➝
	put_prev_task_fair➝
	put_prev_entity➝
	__account_cfs_rq_runtime➝

	ret_from_fork➝
	set_kthread_struct➝
	kthread➝
	smpboot_register_percpu_thread_cpumask➝
	smpboot_thread_fn➝
		schedule➝

==== Scheduler

	task_tick_core

===== CFS

	DEFINE_SCHED_CLASS(fair) = {
		check_preempt_wakeup_fair
			resched_curr_lazy ->

	init_sched_fair_class
	nohz_balance_enter_idle
		on_null_domain
			sched_domain ->
			!rcu_dereference_sched
		idle_cpus_mask
	nohz_run_idle_balance
		_nohz_idle_balance
			idle_cpus_mask



	smpboot_thread_fn
		run_ksoftirqd
			handle_softirqs
				trace_softirq_entry
				h->action
				trace_softirq_exit
				sched_balance_domains
					sched_balance_rq

	select_task_rq
		select_fallback_rq
			cpuset_cpus_allowed_fallback
			do_set_cpus_allowed

		sched_energy_enabled
		find_idlest_cpu
		select_idle_sibling
			select_idle_cpu
				select_idle_core
					sched_idle_cpu(cpu)
					cpumask_test_cpu(cpu, p->cpus_ptr))
					BTW:
						tsk->cpus_ptr = &tsk->cpus_mask;

	schedule
		sched_submit_work
		__schedule➝
			+ hrtick_clear
			trace_sched_switch
			deactivate_task
				dequeue_task
					dequeue_task_fair
						dequeue_entity➝
							update_stats_dequeue->
							update_curr
								rq_clock_task
									assert_clock_updated
									WARNING: CPU: 2 PID: 54342 at kernel/sched/sched.h:1207 assert_clock_updated.isra.74.part.75+0x13/0x20
									SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP);
									rq->clock_task
								sum_exec_runtime -> usage_usec, sum_sched_runtime
								vruntime
								update_min_vruntime
									min_vruntime
								account_cfs_rq_runtime ->
			context_switch
				finish_task_switch
					finish_lock_switch
						raw_spin_rq_unlock_irq(rq);
			raw_spin_rq_unlock_irq
	set_tsk_need_resched
			set_tsk_thread_flag
				task_thread_info
				set_ti_thread_flag
			TIF_NEED_RESCHED
		need_resched
			tif_need_resched
				TIF_NEED_RESCHED TIF_NEED_RESCHED_LAZY

		sched_update_worker

		CONFIG_PREEMPT_DYNAMIC
			preempt_dynamic_mode
			cat /sys/kernel/debug/sched/preempt

			preempt_model_none
			preempt_model_voluntary
			preempt_model_full

			might_sleep

			cond_resched
				__cond_resched
					should_resched
					preempt_schedule_common
						preempt_latency_start
							get_lock_parent_ip
								in_lock_functions
									__lock_text_start
									__lock_text_end
							trace_preempt_off ->
						preempt_latency_stop 
							trace_preempt_on ->
						need_resched
							tif_need_resched ->

		got_nohz_idle_kick
			NOHZ_KICK_MASK

		task_sched_runtime
			update_curr_fair
				update_curr
					delta_exec = now - curr->exec_start
					statistics.exec_max
					curr->sum_exec_runtime += delta_exec;
					update_min_vruntime
					trace_sched_stat_runtime
					cgroup_account_cputime
						cpuacct_charge
							usages
					account_group_exec_runtime
						sum_exec_runtime
					account_cfs_rq_runtime	 delta_exec ...
						__account_cfs_rq_runtime
							cfs_rq->runtime_remaining -= delta_exec
							assign_cfs_rq_runtime
								sched_cfs_bandwidth_slice
								__assign_cfs_rq_runtime
									start_cfs_bandwidth ->
									cfs_b->runtime -= amount
									cfs_rq->runtime_remaining += amount;
							resched_curr_lazy ->

	echo distribute_cfs_runtime > set_ftrace_filter

	set_ftrace_notrace

	echo function > current_tracer

	echo nop > current_tracer

	echo 1 > tracing_on
	echo 0 > tracing_on
	echo > trace

	sched_create_group
		alloc_fair_sched_group
			init_cfs_bandwidth
				default_cfs_period
				period_timer
				sched_cfs_period_timer < __run_hrtimer
					cfs_bandwidth
					hrtimer_forward_now->
					do_sched_cfs_period_timer
						throttled = !list_empty(&cfs_b->throttled_cfs_rq)
						cfs_b->nr_periods += overrun
						__refill_cfs_bandwidth_runtime
							cfs_b->runtime = cfs_b->quota;
						cfs_b->nr_throttled += overrun;
						distribute_cfs_runtime
							cfs_rq
							list_for_each_entry_rcu
							cfs_b->throttled_cfs_rq

							cfs_rq->runtime_remaining += runtime;
							cfs_b->runtime -= runtime

							cfs_rq->runtime_remaining > 0
							unthrottle_cfs_rq ^
				hrtimer_init CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED

				hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL)

	snd_hrtimer_callback
		hrtimer_resolution
		drift = ktime_sub(hrt->base->get_time(), hrtimer_get_expires(hrt));
		snd_timer_interrupt

	task_struct
		..> task_group
		sched_entity
			-> cfs_rq
				rq
				load_weight
		sched_rt_entity
		sched_dl_entity

	rq
		-> cfs_rq

	task_group
		-> cfs_rq
		cfs_bandwidth
			throttled_cfs_rq

	"throttled_usec" = throttled_time

	cpu.cfs_period_us
	cpu.cfs_quota_us
	cpu.shares
	"cpu.stat"
		cpu_cfs_stat_show
			nr_periods
			nr_throttled
			throttled_time

	"cfs_quota_us"
		tg_set_cfs_quota
			tg_set_cfs_bandwidth
				cfs_bandwidth->quota
				start_cfs_bandwidth
					cfs_b->period_active = 1
					hrtimer_forward_now ->
					hrtimer_start_expires ->
				cfs_bandwidth_usage_inc
				__refill_cfs_bandwidth_runtime ->
				cfs_bandwidth_usage_dec
				unthrottle_cfs_rq ->

	sched_cpu_deactivate
		set_rq_offline
			rq_offline
				...
	cpu_attach_domain
		rq_attach_root
			set_rq_offline ->

	wake_up_state:
		try_to_wake_up kernel/sched/core.c
			ttwu_state_match
			p == current
				trace_sched_waking
				trace_sched_wakeup
			ttwu_runnable
				task_on_rq_queued
					on_rq == TASK_ON_RQ_QUEUED
				ttwu_do_wakeup
					trace_sched_wakeup
					__state = TASK_RUNNING

			trace_sched_waking
			ttwu_stat
			ttwu_queue
				ttwu_queue_wakelist
					TTWU_QUEUE
					ttwu_queue_cond

					__ttwu_queue_wakelist
						llist_add
							llist_add_batch
						sched_clock_cp >
							sched_clock
				ttwu_do_activate
					if (p->sched_contributes_to_load)
						rq->nr_uninterruptible--;
					activate_task ->
						enqueue_task(rq, p, flags);
						p->on_rq = TASK_ON_RQ_QUEUED;
						? inc_nr_running
					ttwu_do_wakeup ->

	... try_to_wake_up ...
	__flush_smp_call_function_queue
		sched_ttwu_pending
			ttwu_do_activate
				ttwu_do_wakeup

	< ttwu_do_activate ttwu_runnable

	< wake_up_process wake_up_state default_wake_function

	...
		try_to_freeze
			try_to_freeze_unsafe
				__refrigerator
					set_current_state(TASK_FROZEN)
					freezing
						freezing_slow_path
							cgroup_freezing
								task_freezer
					schedule
	...
		kthread_freezable_should_stop
			__refrigerator

	echo 'func * =pf' > control; dmesg -tw | cut -f1 -d: | while read a;do echo $a; echo "func $a =_" > control; done

	dmesg -Tw | grep -v -e 'CDROM\|ADDRCONF\|promiscuous'

	WQ_WATCHDOG
		workqueue.watchdog_thresh
		wq_watchdog_thresh
		wq_watchdog_timer_fn
			show_all_workqueues
				show_one_workqueue
					show_pwq
				show_one_worker_pool
					pr_cont_worker_id

	/sys/module/workqueue/parameters/watchdog_thresh

	workqueue


	PF_KTHREAD

	kernel_init_freeable
		workqueue_init
			worker_pool
			for_each_bh_worker_pool
			for_each_cpu_worker_pool
				cpu_worker_pools
			workqueues
			init_rescuer
				alloc_worker
				&rescuer_thread
					process_scheduled_works
						process_one_work
				kthread_create
				unbound_effective_cpumask ->
				kthread_bind_mask
				cpu_possible_mask
			create_worker ->
		workqueue_init_topology
			for_each_online_cpu
			unbound_wq_update_pwq
				wq_unbound_cpumask
				alloc_unbound_pwq
					init_pwq
						kthread_init_work->


			workqueue_struct	lkm2
				create_workqueue  ▻
				queue_work
				worker
				cpu_workqueue_struct
					wait_queue_head_t
					work_struct  ▻
					task_struct  ▻

			work_struct lkm2
				DECLARE_WORK
				INIT_WORK	lkm2
				schedule_work  ▻

			create_worker
				format_worker_id
					"kworker/..."
					R - rescue_wq
					u - unpinned
					H - High prio, nice < 0
				pool->node
				kthread_create_on_node
				&worker_thread
				worker_attach_to_pool
					kthread_set_per_cpu
					set_cpus_allowed_ptr
		ret_from_fork
			kthread
				worker_thread
					process_one_work
						trace_workqueue_execute_start
						worker->current_func
						trace_workqueue_execute_end

		kthread_create
			returns task_struct
			NUMA_NO_NODE
			kthread_create_on_node
				with real node is called from kswapd_run kcompactd_run ...
				__kthread_create_on_node
					kthread_create_info
					kthread_create_list
					wake_up_process(kthreadd_task);

					kthreadd
						set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD));
						kthread_create_list
						create_kthread ->
			INIT_WORK
				x keventd_create_kthread
					kernel_thread

			create_workqueue kernel/workqueue.c
				__create_workqueue
					x create_workqueue_thread
						kthread_create ->
						worker_thread
							x run_workqueue
								work->func

			alloc_workqueue
				__alloc_workqueue
					alloc_and_link_pwqs
						apply_workqueue_attrs_locked
							wq_unbound_cpumask
							apply_wqattrs_prepare ->
			schedule_work ltd lkm2
				system_wq
				work_struct  ▻
				queue_work
					queue_work_on
						x wq_per_cpu(wq, cpu)
						__queue_work
							cpu = wq_select_unbound_cpu(raw_smp_processor_id());
							insert_work
								set_wq_data
							&cwq->more_work
		run_scheduled_work
			__run_work

	workqueues
		i40e Intel 40 GB Ethernet
		cryptd
		MEM_RECLAIM
		WQ_UNBOUND
		WQ_MEM_RECLAIM

		xfs_init_mount_workqueues

		xxx
				INIT_DELAYED_WORK(&pag->pag_blockgc_work, xfs_blockgc_worker);

				m_blockgc_wq

			workqueue_struct
				pool_workqueue

	wq_watchdog_timer_fn
		show_workqueue_state
			"Showing busy workqueues and worker pools:"
			workqueues
			pool_workqueue
			show_pwq
				pr_cont_work
			pr_cont_pool_info

	pool_workqueue

	git config remote.pushDefault my

	unshare
		sys_unshare
			ksys_unshare
				unshare_fs

	setns
		sys_setns
			commit_nsset
				set_fs_root

	kcmp
		sys_kcmp
			kcmp_lock
			kcmp_ptr
			kcmp_unlock

	hwlat_sample -> hwlat_entry
	hwlat_tracer
		hwlat_tracer_init
		hwlat_tracer_start

	init_tracer_tracefs
		trace_time_stamp_mode_fops
			tracing_time_stamp_mode_open
				tracing_time_stamp_mode_show
					ring_buffer_time_stamp_abs
		"tracing_on"
			rb_simple_fops
				rb_simple_write
					start
						hwlat_tracer_start ->
					hwlat_tracer_stop
						stop_single_kthread
				rb_simple_read
					tracer_tracing_is_on
						ring_buffer_record_is_on
			tracing_set_tracer
				hwlat_tracer_reset
					hwlat_tracer_stop
		"current_tracer"
		set_tracer_fops
			tracing_set_trace_write
				tracing_set_tracer ->


	read_sum_exec_runtime
		sum_exec_runtime

	?
		cpuacct.usage

	__update_load_avg_cfs_rq
		___update_load_avg

	cgroup
	cgroup_root
	cgroup_subsys

	cgroup_base_files
		"cpu.stat"
			cpu_stat_show
				cgroup_base_stat_cputime_show
					usage_usec < sum_exec_runtime

	cpus_mask, __do_set_cpus_allowed family

	proc_pid_status
		task_cpus_allowed
			cpus_mask
	set_cpus_allowed_ptr !!
		affinity_context
		__set_cpus_allowed_ptr
			__set_cpus_allowed_ptr_locked
				user_cpus_ptr
				task_cpu
					task_thread_info
					cpu
				__do_set_cpus_allowed ->
					>set_cpus_allowed ->
						set_cpus_allowed_common
							cpumask_copy(&p->cpus_mask, new_mask);
							p->cpus_ptr = new_mask
							p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
							SCA_MIGRATE_...
								cpus_ptr
							cpumask_copy(&p->cpus_mask, ctx->new_mask);
					dequeue_task ->

					->set_cpus_allowed
						set_cpus_allowed_common ->
						set_cpus_allowed_dl
				cpumask_any_and_distribute
			update_rq_clock
				clock_update_flags
				rq->clock += delta
				sched_clock_cpu ->
					sched_clock_data
					sched_clock_remote
						sched_clock_local
					sched_clock_stable
					sched_clock

				update_rq_clock_task
			!(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) &&
			affine_move_task
				move_queued_task


	do_taskset
	sched_setaffinity
		affinity_context
		find_process_by_pid
			find_task_by_vpid
		cpumask_copy(user_mask, in_mask)
		security_task_setscheduler
			task_setscheduler
		 __sched_setaffinity
			alloc_cpumask_var
				alloc_cpumask_var_node
					kmalloc_node
			cpuset_cpus_allowed(p, cpus_allowed);
				spin_lock_irqsave(&callback_lock, flags);
				cpumask_copy
					cpu_possible_mask
						__cpu_possible_mask
			cpumask_and(new_mask, in_mask, cpus_allowed);
			__set_cpus_allowed_ptr ->


in the system:
rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x3d/0x175
	ksys_write
		fdget_pos
			mutex_lock
		? lock "sb_writers" sb_writers_name s_writers.rw_sem
		kernfs_fop_write_iter
			mutex_lock
			cpuset_write_resmask
				is_cpuset_online
				css_get
					cgroup_subsys_state
					percpu_ref_get
						percpu_ref_get_many
							this_cpu_add
								__pcpu_size_call
								percpu_add_op
				cpus_read_lock();
					cpu_hotplug_lock
					percpu_down_read
						__percpu_down_read
				mutex_lock(&cpuset_mutex);
				update_cpumask ->
				FILE_EXCLUSIVE_CPULIST
				update_exclusive_cpumask
					exclusive_cpus
					effective_xcpus
					compute_effective_exclusive_cpumask
					remote_cpus_update ->
					update_parent_effective_cpumask
						lockdep_assert_held(&cpuset_mutex);
						spin_lock_irq(&callback_lock);
						partition_xcpus_del
						partition_xcpus_add ->
						w2 update_isolation_cpumasks -> (update_unbound_workqueue_cpumask)
						cpuset_update_tasks_cpumask ->
						update_sibling_cpumasks ->
						notify_partition_change
							cgroup_file_notify
								kernfs_notify
					validate_change
					cpumask_copy
					update_cpumasks_hier
				cpus_read_unlock
					percpu_up_read
						preempt_disable
						rcuwait_wake_up
					cpu_hotplug_lock
		fdput_pos
			__f_unlock_pos
			fdput ->

	migrate_live_tasks
	set_mempolicy
	migrate_enable
		set_cpus_allowed_common ->

	tracing_read_pipe
		print_trace_line
			print_trace_fmt
				trace_print_context
					"comm-pid [cpu]"
				trace_print_lat_context
					lat_print_generic

	date -d @1676347894.530839161 "+%y-%m-%d %T"
	date -d @410645.867062 "+%y-%m-%d %T"
	date -d @1676347894.530839161 "+%s"

	echo raw > trace_options

	export KUBECONFIG=~/kubeconfig
	export HTTPS_PROXY=socks5://localhost:1337
	kubectl get pods -A

	ssh root@192.168.13.14
	root@192.168.13.14: Permission denied (publickey,gssapi-keyex,gssapi-with-mic).

	BasePage._save

	Request.submit

	module_spi_driver
		spi_register_driver
			driver_register

	spi_drv_probe
		sdrv->probe

	CONFIG_MSM_SMD

	cat /sys/kernel/debug/smd/int_stats

	echo 0x37 > /sys/module/smd/parameters/debug_mask
	printf "0x%x\n" $(cat /sys/module/smd/parameters/debug_mask)

	msm_smd_probe
		smd_get_intr_config
		private_intr_config
			smd_modem_irq_handler
				++interrupt_stats[SMD_MODEM].smd_in_count
				handle_smd_irq
					~ notify_modem_smd
						++interrupt_stats[SMD_MODEM].smd_out_count;
						do_smd_probe
				handle_smd_irq_closing_list

	HP G8 192.168.1.247
	00:1f.3 Multimedia audio controller: Intel Corporation Tiger Lake-LP Smart Sound Technology Audio Controller (rev 20)

	HP G4 192.168.1.139
	00:1f.3 Audio device: Intel Corporation Sunrise Point-LP HD Audio (rev 21)

	old

		FP problem
		Linux swi-mdm9x28-wp 3.18.140 #1 PREEMPT Mon Aug 22 09:34:39 UTC 2022 armv7l GNU/Linux

		killall klogd; /etc/init.d/syslog star

	#!/bin/sh

	pw=13
	echo $pw > /sys/class/gpio/export;
	echo high > /sys/class/gpio/gpio$pw/direction

	pw=41
	echo $pw > /sys/class/gpio/export;
	echo high > /sys/class/gpio/gpio$pw/direction

	lcdcs=1008
	echo $lcdcs > /sys/class/gpio/v2/export
	echo high > /sys/class/gpio/v2/gpio$lcdcs/direction

	export LD_LIBRARY_PATH=$PWD

	./NBDevicesConsoleSampleC -sysfspath /sys/class/gpio/v2 -spi /dev/spidev1.1 16 1005 58

	cs=8 # 58
	echo $cs > /sys/class/gpio/export
	echo high > /sys/class/gpio/gpio$cs/direction
	echo $cs > /sys/class/gpio/unexport

	Reset 2P (shut down) - the power cable must be disconnected!!!:
	write to address 0x6A, register 9, value 64:
	/bq25890/w 9 64

	lua5.3

	Add_A_to_C:
		inc C
		inc D
		dec A
		jnz Add_A_to_C
		// D is original A
	Move_D_to_A:
		inc A
		dec D
		jnz Move_D_to_A
		dec B
		jnz Add_A_to_C // B-1 times
		stop

	USE_MDFILE_AS_MAINPAGE = DEVELOPERS.md

	\xb4\x14\x0e\xb7\x1d\x0c\x88\xc5 + 3a78f4e48f3e30cb = \x7f\x24\x30\x38\xf9\xf8\xf0\xff

	__isoc99_scanf@plt
	fexecve

	sed -n 's/\(.*\):\(.*\):.*:.*warning: unused parameter .\(\w*\).*/ex -c "\2\/{\/|exec \\\"normal jwi(void)\3;\\n\\"|x" \1/p' cmake.log > a

	Traceback (most recent call last):
	File "/usr/lib/python3/dist-packages/xdot/ui/window.py", line 171, in update
	current_mtime = os.stat(self.openfilename).st_mtime
	FileNotFoundError: [Errno 2] No such file or directory: 'colors.dot'

	gpiod_to_irq
	nmap tT4 -A -v 192.168.0.100 # -Taggressive Aggressive
	sudo nmap polite

	sys_timerfd_settime
		do_timerfd_settime

	sys_timerfd_create
		anon_inode_getfd  ▻
			__anon_inode_getfd
				__anon_inode_getfile
					alloc_file_pseudo
						alloc_file  ▻
			anon_inode_getfile ->
				alloc_file  ▻
		if isalarm
			alarm_init
				hrtimer_init
				alarmtimer_fired
			timerfd_alarmproc
		! isalarm
			hrtimer_init

	pm_suspend
	enter_state
	suspend_devices_and_enter
		suspend_enter
			pm_wakeup_pending
			3.18 pm_get_active_wakeup_sources
	sysdev_suspend

	input_dev_suspend
		input_dev_release_keys

	pm_print_active_wakeup_sources

	3 sys_epoll_ctl

	ep_remove
	ep_destroy_wakeup_source
		wakeup_source_unregister

	do_epoll_ctl
		ep_insert
			ep_create_wakeup_source
				wakeup_source_register
					wakeup_source_create
					wakeup_source_sysfs_add
					wakeup_source_add
						wakeup_sources
	alarmtimer_init
		wakeup_source_register ->

	ssh ilan@ilan make -C /home/ilan/Leaf/swi-linux-src
	register_chrdev_region
		__register_chrdev_region
	alloc_chrdev_region
		major == 0
		__register_chrdev_region
			find_dynamic_major
	mousedev_init
		mousedev_create
			cdev_init
			cdev_device_add
				cdev_add ->
				device_add

	git-authors

	corbet@lwn.net
	linux-doc@vger.kernel.org
	kernelnewbies@kernelnewbies.org

	NETLINK_AUDIT

	fbtft_display

	ili9341_spi_driver
		ili9341_probe
			ili9341_pipe_funcs
				mipi_dbi_pipe_update
					mipi_dbi_fb_dirty
						MIPI_DCS_WRITE_MEMORY_START

	spi_sync_transfer
		spi_message_init_with_transfers
			spi_message_init
			spi_message_add_tail ->

	msm_spi_request_cs_gpio
		cs_gpios

	spi_register_controller

	pm_generic_runtime_suspend
		runtime_suspend

	msm_spi_request_irq
		msm_spi_qup_irq
			devm_request_irq
			msm_spi_error_irq
			msm_spi_output_irq

	spi_pump_messages
		spi_finalize_current_message
		transfer_one_message
					? spi_transfer_one_message <- transfer_one_message
						transfers
						spi_set_cs true - sets run time chipselect, active high
							msm_spi_set_cs - active low
								SPI_IO_C_FORCE_CS
								SPI_CS_HIGH?
						transfer_one
						spi_set_cs false

	spi_write
		spi_device
		spi_message_init
		spi_message_add_tail
			transfer_list
			list_add_tail
				__list_add
		spi_sync
			__spi_sync
				spi_async_locked
					__spi_validate
					__spi_async
						transfer
							spi_queued_transfer
								pump_messages
								... spi_pump_messages ->
					gpio_free
				wait_for_completion

	process_one_work
		fb_deferred_io_work
			ili9341_update
	fb_info
		creen_base
		screen_size
	fb_fix_screeninfo
		smem_start
		smem_len
	fb_deferred_io_cleanup
		cancel_delayed_work_sync
		fb_deferred_io_page
	a
		readcommand8
			sendCommand
				spiWrite
			spiRead
		Adafruit_ILI9341::begin
		fillScreen
			fillRect
				...
					setAddrWindow
						ILI9341_RAMWR
		writePixel
			drawPixel
				startWrite
					SPI_CS_LOW
				setAddrWindow(x, y, 1, 1);
				SPI_WRITE16(color);
	mipi_dbi_spi_init
		mipi_dbi_typec3_command
			mipi_dbi_typec3_command_read
	virtio_gpu_primary_helper_funcs
		virtio_gpu_primary_plane_update
			drm_atomic_helper_damage_merged
				drm_atomic_helper_damage_iter_init
					drm_helper_get_plane_damage_clips
						drm_plane_get_damage_clips
					drm_rect_equals
	mipi_dsi_dcs_set_page_address
	mipi_dsi_dcs_set_column_address
		MIPI_DCS_SET_COLUMN_ADDRESS
	git log -p --pretty=$'format:%ae\n%ce' -n100 --no-merges --since="1 year ago" -- Documentation/  ":(exclude)Documentation/devicetree/bindings"
	d=https://www.kernel.org/doc/html/latest/trace/index.html
	wget -q -O- https://www.kernel.org/doc/html/latest/core-api/index.html \
		sed 's@<h.>\([^<]\+\).*headerlink.*href="\(#[^"]\+\).*@{{The Linux Kernel/doc|\1|core-api\2}}@p;d'

	wget -q -O- https://www.kernel.org/doc/html/latest/core-api/ | sed 's@.*toctree-l\(3\).*href="\([^"]\+\)">\(.[^<]\+\)<.*@*\1 {{The Linux Kernel/doc|\3|core-api/\2}}@p;d'
	device_link_add
		device_register
	napi
		netif_napi_add
		napi_complete
			napi_complete_done
		e1000e_poll
			napi_complete_done
				gro_normal_list
					netif_receive_skb_list_internal
						__netif_receive_skb_list
							__netif_receive_skb_list_core
								__netif_receive_skb_core
		e1000_clean
	skb_queue_tail
		__skb_queue_tail
	skb_dequeue
		__skb_dequeue
	dev_cpu_dead
		netif_rx_ni
			netif_rx_internal
				enqueue_to_backlog
	MCE
	do_machine_check arch/x86/include/asm/mce.h
		mce_panic
		kill_me_maybe
			memory_failure
	memory_failure_init
		memory_failure

	{	air
		w=wlp2s0
		rfkill unblock 1
		sudo airmon-ng start $w
		sudo airodump-ng ${w}mon

		sudo airodump-ng ${w}mon -d 14:AE:DB:C7:18:52 -c 9 -w izylho
		sudo airodump-ng ${w}mon --bssid 00:11:6B:26:90:AC --channel 11 --write tkip

		sudo airmon-ng stop ${w}mon

		sudo aircrack-ng izylho-01.cap
		sudo aircrack-ng tkip-01.cap

		sudo aireplay-ng -a $target_ap -c $clent --deauth 1

	}

==== Real-Time preemption

https://wiki.linuxfoundation.org/realtime/start[Real-Time Linux]

{

	rt_sched_class
		balance_rt
			pull_rt_task
				push_cpu_stop

	kernel/sched/rt.c
		enqueue_task_rt
			update_stats_wait_start_rt
				__update_stats_wait_start

		set_next_task_rt
			update_stats_wait_end_rt
				__update_stats_wait_end

=====	rt-tests

		cut -f2 -d: cyclictest.log | sort | uniq -c | sort -n
		cyclictest
			tracelimit
			break_thread_value
			timerthread
				timer_settime
				clock_nanosleep
					sys_clock_nanosleep
						hrtimer_nanosleep ->
		oslat
			doit
				frc
		hwlatdetect
			"tracing_cpumask"
				tracing_cpumask_fops
					tracing_cpumask_write
						cpumask_parse_user
							bitmap_parse_user
								memdup_user_nul
								bitmap_parse
			init_hwlat_tracer
				register_tracer
				...
				hwlat_tracer
				hwlat_tracer_start
					hwlat_data
					start_per_cpu_kthreads
						start_cpu_kthread
							kthread_run_on_cpu ->
							kthread_fn ->
					start_single_kthread
										cpu_online_mask
						kthread_create ->
						kthread_fn
							move_to_next_cpu
								cpumask_next
									cpumask_equal(current_mask, current->cpus_ptr))

							save_cpumask
							get_sample
								tracing_thresh
								time_get ->
								sample -> duration -> inner
								outer_sample -> outer_duration
								trace_hwlat_sample
									__buffer_unlock_commit
								max_latency

							init_events
								trace_hwlat_event
									trace_hwlat_funcs
										trace_hwlat_print
											duration,outer_duration ->
											"inner/outer"
											trace_seq_printf
						cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);
							cpu_online_mask
								&__cpu_online_mask
						sched_setaffinity(kthread->pid, current_mask);
				init_tracefs
					tracefs_create_dir
					"hwlat_detector"
					thread_mode_fops
						hwlat_mode_write
							hwlat_tracer_stop->
								hwlat_data.thread_mode
									sample_window
									sample_width
							hwlat_tracer_start->

===== rtla in kernel
	rtla
		???
		timerlat_aa_handler
		timerlat_aa_stack_handler	

		timerlat_main ->
			timerlat_hist_ops
				timerlat_init_hist
					timerlat_alloc_histogram
						timerlat_hist_data
							timerlat_hist_cpu
								irq_count
					tep_register_event_handler
					timerlat_hist_handler
						timerlat_hist_update
							update_max
							max_irq
			timerlat_top_ops
				timerlat_enable
				timerlat_init_top
					timerlat_alloc_top
						timerlat_top_data
							timerlat_top_cpu
								irq_count
					tep_register_event_handler
					timerlat_top_handler
						timerlat_top_update
							update_max
							cur_user
							cur_thread
							cur_irq
	late_initcall
	init_osnoise_tracer
		print_osnoise_headers
		register_tracer
		osnoise_tracer
			osnoise_tracer_stop
				osnoise_workload_stop
					osnoise_unhook_events ^
		timerlat_tracer
			timerlat_tracer_start
				osnoise_workload_start ^
				__osnoise_tracer_start
					start_per_cpu_kthreads
						start_kthread
							osnoise_main
							timerlat_main
								timerlat_top_ops ->
								timerlat_hist_ops ->
								run_tool
									parse_args
										timerlat_hist_parse_args
								hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
								hrtimer_cb_get_time
									get_time
								s.timer_latency = diff
								trace_timerlat_sample
									osnoise_instances
									__trace_timerlat_sample
										trace_buffer_lock_reserve
											__trace_buffer_lock_reserve
												ring_buffer_lock_reserve
													rb_reserve_next_event
								timerlat_irq < __run_hrtimer
									stop_tracing
									time_get ->
									tlat->count++
									s.timer_latency = diff
									trace_timerlat_sample
								wait_next_period
									hrtimer_start
									schedule

	grep -E '[0-9]{5} ns' trace

	trace_osnoise_funcs
		trace_osnoise_print
		trace_osnoise_raw
	
	start_kthread

		osnoise_main
			osnoise_tracer
			grep '' /sys/kernel/debug/tracing/instances/osnoise_*/options/*
			echo 1 > instances/osnoise_trace/options/stacktrace
			run_osnoise

				this_cpu_osn_var
					per_cpu_osnoise_var
				osnoise_options
				osnoise_data
				OSN_IRQ_DISABLE
				int_count = set_int_safe_time
				set_int_safe_time
					time_get
						trace_clock_local
							sched_clock
					int_counter
				interference = int_count - last_int_count;
				hw_count++ if (!interference)
				s.noise = time_to_us(sum_noise);
				s.runtime = time_to_us(total);
				s.max_sample = time_to_us(max_noise);
				s.un_osnoise
				time_sub

				trace_sample_threshold "sample_threshold"
				osnoise_stop_tracing
					<
						run_osnoise
						timerlat_irq
						timerlat_main
						timerlat_fd_read
				record_osnoise_sample
					trace_osnoise_sample
						hw_count
					__record_osnoise_sample
						trace_buffer_unlock_commit_nostack
			osnoise_sleep(false);
				ktime_add_us
				schedule_hrtimeout

			trace_osnoise_sample(&s);
				__trace_osnoise_sample
			notify_new_max_latency




		trace_instance_init
			create_instance
				libtracefs
				tracefs_instance_create
		timerlat_aa_register_events
			tracefs_event_enable
		tracefs_instance

		hwnoise_main
			run 
			osnoise_top_ops ->
				top_main_loop ->
			threshold
			x osnoise_top_main
				...
				osnoise_params
					threshold|
				trace_instance_start
				osnoise_init_top
					&osnoise_top_handler
						hw_count
						nmi_count
				osnoise_top_apply_config
					if (params->mode == MODE_HWNOISE) {
						retval = osnoise_set_irq_disable(tool->context, 1);
							"OSNOISE_IRQ_DISABLE" -> OSN_IRQ_DISABLE
							osnoise_options_set_option
				tracefs_iterate_raw_events
				&collect_registered_events
				trace_is_off
					!tracefs_trace_is_on
				save_trace_to_file
				osnoise_destroy_tool
					trace_instance_destroy
						destroy_instance
							tracefs_instance_destroy
							tracefs_instance_free
===== RT user tools, rtla

	for t in osnoise timerlat;do for m in top hist; do sudo ./rtla $t $m -D -d 10s -H 1 -c 2; done; done


	main rtla
		new
		nr_cpus

	- rtla_usage
	- run_command
		timerlat_main ->
		osnoise_main (um)
			run_tool
				parse_args
					osnoise_hist_parse_args
						new: 
							common_init
							common_parse_args
			osnoise_top_ops
				osnoise_top_parse_args
					stop_us
					trace_output
				osnoise_top_apply_config
					osnoise_apply_config
						new? osnoise_set_workload
						threshold
						osnoise_set_tracing_thresh ->
						common_apply_config
							osnoise_set_workload
								osnoise_options_set_option
								"OSNOISE_WORKLOAD"
									k: OSN_WORKLOAD
										? osnoise_workload_start ->
											start_per_cpu_kthreads
					osnoise_set_stop_us
				top_main_loop
			osnoise_top_main
			osnoise_hist_main
				osnoise_hist_apply_config
					osnoise_apply_config ->
					-a stop_us
					osnoise_set_stop_us
						"/sys/kernel/tracing/osnoise/stop_tracing_us" -> osnoise_stop_tracing_in
						stop_tracing
				osnoise_init_trace_hist
					"sample_threshold"
				tracefs_iterate_raw_events
				&collect_registered_events
				osnoise_trace_is_off
					tracefs_trace_is_on
				osnoise_print_stats
					osnoise_top_header
					osnoise_top_print
						trace_seq_printf
			osnoise_set_tracing_thresh /sys/kernel/debug/tracing/tracing_thresh
				"tracing_thresh"
				osnoise_write_ll_config
					tracefs_instance_file_write
			osnoise_init_trace_tool
				tracefs_event_enable

		osnoise_hist_update_multiple
			update_max(&data->hist[cpu].max_sample, &duration);

		osnoise_top_handler
			max_noise
			update_max(&cpu_data->max_noise, &val);
			update_max(&cpu_data->max_sample, &val);


	__print_ns_without_secs

	timerlat_hist_main
		timerlat_hist_bpf_main_loop
			timerlat_hist_bpf_pull_data
				timerlat_bpf_get_hist_value
					get_value
	del nr_cpus

Consiolidate duplicated lines into save_trace_to_file().

===== kernel trace

kernel/trace/trace_osnoise.c
	osnoise_tracer
	osnoise_tracer_init
	-osnoise_tracer_start
		osnoise_workload_start
			osnoise_hook_events
				hook_thread_events
					register_trace_sched_switch
						__DECLARE_TRACE_COMMON
							sched_switch
					&trace_sched_switch_callback
						thread_entry
							delta_start
						thread_exit
							delta_start
					register_migration_monitor
								trace_thread_noise "thread_noise"
				hook_irq_events
					register_trace_irq_handler_entry
						trace_irq_handler_entry
							handler_entry
					->trace_irqentry_callback
						osnoise_trace_irq_entry
							set_int_safe_time(osn_var, &osn_var->irq.delta_start);
					register_trace_irq_handler_exit
						&trace_irqexit_callback
							osnoise_trace_irq_exit
								trace_irq_noise
									__get_str
										__get_dynamic_array
								"duration" -> timer_irq_duration
								get_int_safe_duration
					osnoise_arch_register
						register_trace_irq_work_entry(trace_intel_irq_entry, NULL);
						"irq_work"
						register_trace_irq_work_exit(trace_intel_irq_exit, "irq_work");
						trace_intel_irq_exit ->

						register_trace_local_timer_entry
						register_trace_local_timer_exit

				hook_softirq_events
					trace_softirq_exit_callback
						duration =
						get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
						trace_softirq_noise
						"duration"
				unhook_softirq_events
					trace_softirq_exit_callback ^
			osnoise_unhook_events
				unhook_softirq_events ^

===== RT sched

	sched_rt_period_timer
	Scheduling classes
		SCHED_FIFO
		SCHED_RR

	sched/rt.c implements SCHED_FIFO and SCHED_RR
		sched_rt_runtime_exceeded
			sched_rt_rq_dequeue

	chrt/cyclictest
		sched_setscheduler
			sched_param
			_sched_setscheduler
				__sched_setscheduler
					cpuset_lock
					scx_check_setscheduler
					__setscheduler_params


	ktime
		CONFIG_DEBUG_TIMEKEEPING
			timekeeping_check_update
			timekeeping_get_delta
				read_seqcount_begin
				tk_clock_read
				read_seqcount_retry
					do_read_seqcount_retry
						do___read_seqcount_retry
							s->sequence != start
				clocksource_delta
		ktime_set
		ktime_get
			&tk_core.timekeeper
			read_seqcount_begin
			timekeeping_get_ns
				timekeeping_get_delta
				timekeeping_delta_to_ns
			read_seqcount_retry ->
			ktime_add_ns

		ktime_get_real
			getnstimeofday
				timekeeper
					clocksource


	CONFIG_PREEMPT_VOLUNTARY
		might_sleep	300+ uses
			__might_sleep
				__might_resched
					rcu_sleep_check
			+ might_resched
				_cond_resched
					_cond_resched
						should_resched
						__cond_resched
							__schedule  ▻
	CONFIG_PREEMPT
		resume_kernel
		retint_kernel
			preempt_schedule_irq
				__schedule  ▻

		preempt_check_resched
			preempt_schedule
				__schedule  ▻

	request_any_context_irq
		request_threaded_irq  ▻

	CONFIG_GENERIC_HARDIRQS
	request_irq	ldt
		request_threaded_irq
			irq_desc
				affinity_hint
			irq_to_desc
				sparse_irqs
					< irq_insert_desc
			irq_chip_pm_get
				irq_get_pm_device
				pm_runtime_resume_and_get

			irq_settings_can_request

			__setup_irq
				register_irq_proc  ▻

			thread_fn

				setup_irq
					__setup_irq ->
						desc->action = irqaction
						irq_desc
						irq_activate
							irq_domain_activate_irq
								__irq_domain_activate_irq
						irq_pm_install_action
						register_irq_proc ->
						register_handler_proc ->
						__enable_irq
							irq_startup
								irq_data_get_affinity_mask
								__irq_startup
									irq_state_set_started
										IRQD_IRQ_STARTED
						IRQF_NOBALANCING irq_settings_set_no_balancing IRQD_NO_BALANCING
							_IRQ_NO_BALANCING
						IRQF_PERCPU > IRQD_PER_CPU

			irq_thread_fn
	ret_from_fork > kthread > irq_thread > irq_thread_fn
	interrupt_init_v2_hw
		devm_request_irq
			devm_request_threaded_irq
				devm_irq_release
				devres_alloc
				request_threaded_irq  ▻

	x smp_apic_timer_interrupt
		local_apic_timer_interrupt

	rt:
	cpupri_init
	check_preempt_equal_prio
		cpupri_find
			cpupri_find_fitness
				__cpupri_find
}

== Storage

<<_blk_mq>>

{	Storage (file - disks)

=== UAPI
	libc:
		aiocbp
		aio_read aio_write
		io_setup
		io_submit aio_cancel aio_error
		aio_init
		aio_return
		aio_suspend
		aio_fsync
		io_cancel io_destroy io_getevents
	lio_listio(3)
	sys_io_submit
		iocb
			aio_data
		kioctx
			lookup_ioctx
		kill_ioctx
			io_submit_one
				aio_write
				aio_read
					kiocb
					aio_setup_rw
						import_iovec  ▻
					aio_rw_done
						aio_complete_rw
							aio_complete
				aio_poll

=== FS

==== VFS
		Virtual Filesystem (VFS)
		 Inode Caches and Interaction with Dcache
		 Filesystem Registration/Unregistration
		 File Descriptor Management
		 File Structure Management
		 Superblock and Mountpoint Management
		 Example Virtual Filesystem: pipefs
		 Example Disk Filesystem: BFS
		 Execution Domains and Binary Formats

		12. The Virtual Filesystem
		      The Role of the Virtual Filesystem (VFS)
		      VFS Data Structures
		      Filesystem Types
		      Filesystem Mounting
		      Pathname Lookup
		      Implementations of VFS System Calls
		      File Locking

		sys_link
			sys_linkat
				lookup_create
		sys_unlink
			sys_unlinkat
		sys_waitpid
			sys_wait4
		sys_close
			close_fd
				filp_close file
					filp->f_op->flush
					dnotify_flush
					fput
						delayed_fput
							__fput
								file_free
									file_free_rcu

		sys_creat
			do_sys_open

		sys_open
			ksys_open
			do_sys_open
				do_sys_openat2
				struct filename
					file_open_name
				build_open_flags
				fd = get_unused_fd_flags
					__alloc_fd
				fsnotify_open
					fsnotify_parent
					fsnotify
				get_unused_fd
				do_filp_open
					set_nameidata
					path_openat
						struct file
						get_empty_filp
							__get_empty_filp
						do_last
							vfs_open
					x open_namei 150 LOC
						path_lookup_open   ▻
						may_open < do_open < path_openat < do_filp_open
							vfs_permission
								permission
									security_inode_permission
										security_ops->inode_permission
					nameidata_to_filp
						__dentry_open
							fops_get
				fd_install
					fd
					struct file
					__fd_install
						files_struct
						files_fdtable
						rcu_dereference((files)->fdt)
						rcu_assign_pointer
			getname
				do_getname
					strncpy_from_user
			filp_open
				do_filp_open
				nameidata
				open_namei
					open_namei_create
						vfs_create
							security_inode_create
							create
								ext3_create (ext3_dir_inode_operations)
							fsnotify_create
								fsnotify(inode, FS_CREATE, dentry->d_inode, FSNOTIFY_EVENT_INODE, dentry->d_name.name, 0);
					path_lookup
						do_path_lookup  ▻
						__emul_lookup_dentry
							path_walk
								link_path_walk
				dentry_open
				vfs_open
					do_dentry_open
						fops_get
						open
							ext4_file_open

		linux/fs

		sys_read vfs
			ksys_read
				struct fd
				fdget_pos
					__fdget_pos
						fd  ▻ file
						__fdget
							__fget_light
								struct files_struct
								current->files
								__fcheck_files
									struct fdtable
								__fget
						struct file
							f_path
								path_get
								struct path
									struct vfsmount
									struct dentry
								dentry
									d_seq
									d_name
									d_dname
									d_path
									simple_dname
										prepend
							file->f_inode->i_ino
			fdget .. fdput  ▻
				fget_light
			vfs_read
				fsnotify_access
					FS_ACCESS
					fsnotify  ▻
				security_file_permission (file, MAY_READ)
					security_ops->file_permission
				file->f_op->read
					mtd_read >
				do_sync_read ./fs/read_write.c
					filp->f_op->aio_read  ▻
					generic_file_aio_read	aio_read	./mm/filemap.c
						generic_file_direct_IO
							direct_IO
						do_generic_file_read
							do_generic_mapping_read
								readpage
						xx __generic_file_aio_read
							do_generic_mapping_read
								__lock_page
									__wait_on_bit_lock
										sync_page
											block_sync_page
												blk_backing_dev_unplug
													generic_unplug_device --  fire a request queue
														__generic_unplug_device
															do_ide_request ./drivers/ide/ide-io.c <- somewere
																ide_do_request <- ide_intr
																	start_request
																		ide_do_rw_disk  ▻
																	elv_queue_empty

			fdput
				fd_file
				fput
		sys_write
			fdget
				atomic_long_inc_not_zero
			vfs_write
				__vfs_write
					file->f_op->write
					new_sync_write
						filp->f_op->write_iter
							generic_file_write_iter
								__generic_file_write_iter
				x do_sync_write
					filp->f_op->aio_write
				fsnotify_modify
					FS_MODIFY
					fsnotify  ▻

			fput_light
		wchar

		http://www.tldp.org/LDP/tlk/fs/filesystem.html#tth_sEc9.1

		lfs  ▻

		== lfs

		vfs  ▻

==== CIFS
	init_smb_fs
		.. smb_file_operations
		smb_fs_type
	init_cifs
		.. cifs_file_ops
		cifs_fs_type
			cifs_get_sb
				cifs_read_super
					cifs_mount
						cifs_find_tcp_session

==== NFS
		sys_nfsservctl		fs/nfsctl.c
			do_open
				do_kern_mount("nfsd", 0, "nfsd", NULL);
		nfs_fhget
			nfs_file_operations
				nfs_flock

==== EXT3

		vfs_iter_read	include/linux/fs.h
		iov_iter	include/linux/uio.h

		ext3_file_operations file_operations
				ext3_ioctl
					FITRIM:
						ext3_trim_fs
							ext3_trim_all_free
								sb_issue_discard
									blkdev_issue_discard
										REQ_DISCARD
										???
											sd_init_command
												REQ_DISCARD
												sd_setup_discard_cmnd
													UNMAP
	ext3_commit_super
		sync_dirty_buffer

	ext3_new_blocks
	ext3_journal_dirty_metadata
		__ext3_journal_dirty_metadata
			journal_dirty_metadata
	iput
		iput_final
			evict
				ext3_evict_inode

	sync_filesystem
		ext2_sync_fs
			ext2_sync_super
		ext3_sync_fs
			journal_start_commit
		ext4_sync_fs
			ext4_sb_bread

		ext2_file_operations
		ext2_fsync
			generic_file_fsync
				__generic_file_fsync
					sync_mapping_buffers
						private_list
						fsync_buffers_list
							WRITE_SYNC
							write_dirty_buffer
								submit_bh
								.. end_buffer_write_sync
		ext3_sync_file
			filemap_write_and_wait_range  ▻
		ext4_sync_file

		ext4_da_write_end
			generic_write_end
				block_write_end
					__block_commit_write

	SyS_fanotify_mark > fanotify_mark_add_to_mask > fsnotify_set_inode_mark_mask_locked > igrab > _raw_spin_lock
		__raw_spin_lock

		do_raw_spin_lock
                        arch_spin_lock
				queued_spin_lock
					atomic_try_cmpxchg_acquire
						raw_atomic_try_cmpxchg_acquire
							arch_atomic_try_cmpxchg
					queued_spin_lock_slowpath
						set_locked
			debug_spin_lock_before
				SPIN_BUG_ON
					spin_bug
						owner_cpu
						spin_dump
			__spin_lock_debug
			debug_spin_lock_after

	__mark_inode_dirty
		ext3_dirty_inode
			ext3_mark_inode_dirty
				ext3_reserve_inode_write
					__ext3_journal_get_write_access
						journal_get_write_access
							journal_add_journal_head
							do_get_write_access
							journal_put_journal_head

	new 3.17
		usbip
			vhci_hcd_init
			usbip_host_init
			usbip_core_init
		F_SEAL_SHRINK
		memfd_create

	sparse
		cgcc
		make C=1 all
	current:
		pinctrl

		kmalloc ->
		devm_kcalloc
			devm_kmalloc_array
				devm_kmalloc
		devm_kzalloc
			__GFP_ZERO
			devm_kmalloc
				alloc_dr
				...	__kmalloc_track_caller
						kmalloc_slab
						slab_alloc ->
				devres
			... devm_kzalloc_release
			devres_add
				add_dr
					devres_log
					list_add_tail devres_head
		devm_ioremap_resource (was devm_request_and_ioremap)
			__devm_ioremap_resource
				devm_request_mem_region
					__devm_request_region
						devres_alloc
						devm_region_release
							__release_region
								free_resource
									kfree
						__request_region
						devres_add
					~ devm_release_mem_region
						__devm_release_region
							__release_region
							devres_destroy
							devm_region_release
				__devm_ioremap
					devres_alloc_node
					devm_ioremap_release
					ioremap
		?
			devm_ioremap
				__devm_ioremap ->
				devres_alloc
					alloc_dr ->
				ioremap  ▻
				devres_add
				~ devm_iounmap
					devm_ioremap_release
						iounmap  ▻
					devres_destroy
						devres_remove
						devres_free
					iounmap  ▻

==== ext4
			EXT4_GET_MB_CACHE
				((ext4_sb_info *)inode->i_sb->s_fs_info)->s_mb_cache

			ext4_put_super
				ext4_xattr_destroy_cache
					mb_cache_destroy

			ext4_fill_super
				s_mb_cache = ext4_xattr_create_cache
					mb_cache_create
				ext4_xattr_destroy_cache  ▻

		ext4_file_operations file_operations
			ext4_file_open,
			do_sync_read  ▻
			x ext4_file_write < aio_write
			ext4_file_read_iter
			ext4_file_write_iter
				generic_file_aio_write
					x __generic_file_aio_write
						x generic_file_buffered_write
						__generic_file_write_iter
							generic_perform_write
								write_begin
								iov_iter_copy_from_user_atomic
									__copy_from_user_inatomic
				generic_write_sync
					vfs_fsync_range

		ext3_writeback_aops
			B ext3_readpage
				mpage_readpage
					do_mpage_readpage

		The Ext2 and Ext3 Filesystems
		      General Characteristics of Ext2
		      Ext2 Disk Data Structures
		      Ext2 Memory Data Structures
		      Creating the Ext2 Filesystem
		      Ext2 Methods
		      Managing Ext2 Disk Space
		      The Ext3 Filesystem

		init_ext4_fs
		init_ext3_fs
			register_filesystem
			ext3_fs_type
				x ext4_get_sb	get_sb
				x ext3_get_sb	get_sb
					x get_sb_bdev
						sget
						super_block

		ext3_file_operations
			do_sync_read  ▻

=== block devices

	genhd_device_init
		subsystem_register block_subsys

	linux/blkdev.h

	buffer_head

	block_device
		gendisk *    bd_disk

	block_device_operations

	unregister_blkdev

==== gendisk

		blk_mq_requeue_request
			__blk_mq_requeue_request
				...
					requeue

		include/linux/genhd.h
		partition
		gendisk
			capability
				disk_capability_show
					gendisk
			(
				brd_alloc
					flags |= GENHD_FL_SUPPRESS_PARTITION_INFO
				sd, ide, md: GENHD_FL_EXT_DEVT
			)

			hd_struct
			block_device_operations->
			request_queue request_fn_proc
				struct request
					bio
						io_vec

			alloc_disk <- floppy_init, ide_cd_probe,
				alloc_disk_node
					kmalloc_node
			put_disk
			add_disk
						blk_register_region
							kobj_map bdev_map
						register_disk fs/partitions/check.c
						blkdev_get
							__blkdev_get
								rescan_partitions
									check_partition
									blk_register_queue
									class_register sd_disk_class
		?
			blk_put_request

			blk_init_queue
				blk_init_queue_node

			struct request
				end_request
					blkdev_dequeue_request
						elv_dequeue_request
							list_del_init

==== Page Cache

	do_fsync .. schedule
		sys_fdatasync
		do_fsync
		vfs_fsync
			vfs_fsync_range
		ovl_fsync fs/overlayfs/file.c
			vfs_fsync_range
		xfs_file_fsync
			trace_xfs_file_fsync
			xfs_ilock
			xfs_iunlock

		file_write_and_wait_range
			__filemap_fdatawait_range
			__filemap_fdatawrite_range ->
				do_writepages -->
					writepages
						xfs_vm_writepages
							iomap_writepages
								write_cache_pages
								iomap_submit_ioend
		write_cache_pages
			trace_wbc_writepage
		iomap_writepage_map
		submit_bio
			generic_make_request
				do_make_request
					make_request_fn
		blk_mq_make_request
			rq_qos_throttle
				__rq_qos_throttle
					throttle
		wbt_wait @ throttle @ wbt_rqos_ops	 writeback throttling.
		__wbt_wait
			&wbt_inflight_cb
			rq_qos_wait
				prepare_to_wait_exclusive_return
					__wake_up_common
						rq_qos_wake_function
				rq_qos_wake_function
					->wbt_inflight_cb
						rq_wait_inc_below
						get_limit
							limit = rwb->wb_background;
							limit = rwb->wb_normal;
					data->got_token = true;
				!data.got_token
				io_schedule ->

	io_schedule
		schedule
			__schedule
				trace_event_raw_event_sched_switch


	wbt_enable_default
		wbt_init
			wbt_rqos_ops
				wbt_done
			wb_timer_fn
				scale_up
					calc_wb_limits
						wb_normal
						wb_background
				scale_down
					calc_wb_limits ^
			wbt_queue_depth_changed
				wbt_update_limits
					calc_wb_limits ^
	"wbt_lat_usec"
	queue_wb_lat_entry
		queue_wb_lat_store
			wbt_init ^
			wbt_set_min_lat
				wbt_update_limits ^


		do_idle cpuidle_enter cpuidle_enter_state
			ret_from_intr do_IRQ handle_irq handle_edge_irq handle_irq_event
				nvme_irq
					nvme_process_cq nvme_handle_cqe nvme_end_request
				?? end_io=abort_endio?
				blk_mq_free_request
				rq_qos_done
				__rq_qos_done
				wbt_done
					__wbt_done
						wbt_rqw_done
							wake_up_all
								__wake_up
				__wake_up_common_lock
				__wake_up_common
				rq_qos_wake_function
				wake_up_process try_to_wake_up ttwu_do_wakeup
				trace_sched_wakeup

		sys_fsync  ▻
		fs/sync.c
			sys_sync  ▻
			sys_fdatasync
				do_fsync
			sys_fsync  ▻ lkm2
				do_fsync  ▻
			sys_sync_file_range
				filemap_fdatawrite_range
					__filemap_fdatawrite_range ->

		address_space address_space_operations
			dirty_folio <- set_page_dirty
			nfs_file_aops
			def_blk_aops
				blkdev_readpage
					B block_read_full_page
				blkdev_writepage
					B block_write_full_page
						__block_write_full_page
		hfs_file_operations
			file_fsync	fs/sync.c
				write_inode_now	fs/fs-writeback.c
					mapping_cap_writeback_dirty
					writeback_single_inode  ▻

		sysrq_handle_sync
		orderly_poweroff
			emergency_sync
				do_sync_work
					sync_filesystems;

		sys_sync	lkm2
			wakeup_flusher_threads
				bdi_writeback_all
					bdi_alloc_queue_work
						bdi_work_init
						bdi_queue_work
							wake_up_process(wb->task);
								x --> bdi_start_fn
			sync_filesystems
				__sync_filesystem
					writeout_quota_sb
						quota_sync
					writeback_inodes_sb
						bdi_start_writeback
							bdi_alloc_queue_work  ▻
					sync_quota_sb
						quota_sync
					sync_inodes_sb
						bdi_sync_writeback
						wait_sb_inodes
					sb->s_op->sync_fs
	return __sync_blockdev(sb->s_bdev, wait);

			x do_sync
				x wakeup_pdflush
				x	pdflush
				x	__pdflush
				x sync_inodes
				x	__sync_inodes
						sync_blockdev
							filemap_fdatawrite
				sync_supers
					write_super

		sys_fsync lkm2
			do_fsync
				vfs_fsync lkm2
					vfs_fsync_range
						filemap_write_and_wait_range
							__filemap_fdatawrite_range ->
							filemap_fdatawait_range
								__filemap_fdatawait_range
									wait_on_page_writeback
						fop->fsync
							ext3_sync_file
							ext4_sync_file
							nfs_file_fsync
								nfs_do_fsync
									nfs_wb_all
										nfs_write_mapping
											__nfs_write_mapping
												nfs_writepages
													write_cache_pages
													nfs_writepages_callback
														nfs_do_writepage
															nfs_page_async_flush
																...
							nfs_fsync_dir
		bdi_register_dev
			bdi_register
				kthread_run(bdi_forker_thread ()
					bdi_writeback_thread ( old bdi_start_fn )
						wb_do_writeback  ▻

				wb_do_writeback
					wb_writeback
						writeback_inodes_wb
							requeue_io
							writeback_single_inode
								do_writepages lkm2
									address_space_operations
									mapping->a_ops->writepages
										ext4_da_writepages
											write_cache_pages
											__mpage_da_writepage
									generic_writepages
										write_cache_pages
											...
										__writepage
											mapping->a_ops->writepage
												ext4_writepage
								write_inode
									inode->i_sb->s_op->write_inode
										ext3_write_inode
										ext4_write_inode
								inode_sync_complete
								inode_wait_for_writeback
					wb_check_old_data_flush
						wb_writeback

==== bio

	func 			trace point
	blk_mq_submit_bio 	block_bio_queue
	blk_mq_submit_bio	block_getrq
	blk_account_io_start	block_io_start
	blk_add_rq_to_plug	block_plug
	blk_mq_commit_rq	block_unplug
	nvme_setup_cmd
	blk_mq_start_request	block_rq_issue
	nvme_handle_cqe		nvme_sq
	nvme_complete_rq	nvme_complete_rq
	blk_complete_request	block_rq_complete
	blk_account_io_done	block_io_done
	mark_buffer_dirty block_dirty_buffer

	submit_bio_noacct_nocheck block_bio_queue
		trace_block_bio_queue
		__submit_bio
			blk_mq_submit_bio block_getrq
				blk_mq_insert_request
					trace_block_rq_insert
				trace_block_getrq
				blk_account_io_start -> block_io_start
					blk_mq_start_request
						trace_block_rq_issue
						io_start_time_ns = ktime_get_ns

	tracepoint:block:
		block_bio_queue submit_bio_noacct_nocheck __submit_bio
		block_getrq	blk_mq_submit_bio
		block_io_start NEW	blk_account_io_start
		block_rq_issue blk_mq_start_request
		...
		block_rq_complete < blk_complete_request blk_update_request
		block_io_done blk_account_io_done
		block_bio_complete

	blk_stat_add
		io_start_time_ns
		blk_throtl_stat_add

  # echo 'hist:keys=delta.buckets=100,stack.stacktrace:sort=delta' > events/synthetic/block_lat/trigger
  # cat events/synthetic/block_lat/hist

==== blk_mq


	common structs/methods
		blk_mq_alloc_disk
		blk_mq_tag_set
			blk_mq_alloc_tag_set
			blk_mq_free_tag_set
			blk_mq_update_nr_hw_queues
		request
			blk_mq_complete_request
			blk_mq_end_request
		blk_mq_ctx
			__blk_mq_get_ctx
				queue_ctx
				per_cpu_ptr
			&request_queue
			&blk_mq_hw_ctx
		blk_mq_hw_ctx
		blk_mq_queue_map
			blk_mq_map_queues
		request
			blk_mq_start_request
			blk_mq_requeue_request
			blk_mq_set_request_complete
			blk_mq_request_completed
			pdu
				blk_mq_rq_from_pdu
				blk_mq_rq_to_pdu
		request_queue (blkdev.h)
			bdev_get_queue
				bd_queue
			&blk_mq_ctx
			blk_mq_start_stopped_hw_queues
			blk_mq_stop_hw_queues


	__nvme_submit_sync_cmd
	blk_mq_alloc_request_hctx


	nvme_queue_auth_work
		nvme_auth_submit
			__nvme_submit_sync_cmd
				blk_mq_alloc_request
					__blk_mq_alloc_requests
						blk_mq_rq_ctx_init
				blk_mq_alloc_request_hctx
					cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
					blk_mq_rq_ctx_init


	__blk_mq_update_nr_hw_queues
		blk_mq_sysfs_register_hctxs
			blk_mq_register_hctx
				hctx_for_each_ctx
		blk_mq_debugfs_register_hctxs ->

block/blk-mq-debugfs.c

	blk_mq_debugfs_register_hctxs
		queue_for_each_hw_ctx
		blk_mq_debugfs_register_hctx
			blk_mq_debugfs_hctx_attrs
				hctx_state_show
					hctx_state_name

	mq_map blk_mq_queue_map set, msi_desc
		used by
			blk_mq_map_queues
			blk_mq_hw_queue_to_node
			blk_mq_pci_map_queues
			blk_mq_virtio_map_queues
				get_vq_affinity
			blk_mq_map_swqueue
			blk_mq_alloc_tag_set
			blk_mq_free_tag_set
			blk_mq_map_queue_type
			blk_mq_clear_mq_map
			ublk_ctrl_get_queue_affinity
			hisi_sas_queue_command
			map_queues_v2_hw
			virtio_fs
			virtio_fs_ktype_release
			virtio_fs_map_queues
			virtio_fs_setup_vqs
			blk_mq_queue_map

	cpuhp_invoke_callback ->
		hlist_for_each
		trace_cpuhp_multi_enter
		blk_mq_hctx_notify_offline CPUHP_AP_BLK_MQ_ONLINE
			blk_mq_hw_ctx
			!blk_mq_hctx_has_online_cpu Ming Lei, manager Jonathan Brassow
				for_each_online_cpu
				blk_mq_map_queue_type
					hctx_table
			set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
			blk_mq_hctx_has_requests
				blk_mq_all_tag_iter
					__blk_mq_all_tag_iter
						bt_tags_for_each
							bt_tags_iter_data
							sbitmap_for_each_set
								__sbitmap_for_each_set
				blk_mq_has_request
					iter_data->has_rq = true;
				data.has_rq;
		blk_mq_hctx_notify_dead CPUHP_BLK_MQ_DEAD
			rq_list
			blk_mq_cpu_mapped_to_hctx
				blk_mq_map_queue_type
			cpuhp_dead
			list_splice_init
			blk_mq_hctx_clear_pending
				sbitmap_clear_bit(&hctx->ctx_map, bit);
			.. -> dispatch
			blk_mq_run_hw_queue
				blk_mq_hw_queue_need_run
					blk_mq_hctx_has_pending
						blk_mq_sched_has_work
							has_work
				blk_queue_quiesced
					QUEUE_FLAG_QUIESCED
				blk_mq_delay_run_hw_queue
					run_work
						blk_mq_run_work_fn
							blk_mq_run_dispatch_ops
							blk_mq_sched_dispatch_requests ->
				kblockd_mod_delayed_work_on
						mod_delayed_work_on
							__queue_delayed_work ->
				blk_mq_run_dispatch_ops
					__blk_mq_run_dispatch_ops
				blk_mq_sched_dispatch_requests
					__blk_mq_sched_dispatch_requests
						blk_mq_do_dispatch_ctx
							blk_mq_dequeue_from_ctx
								dispatch_rq_from_ctx
									rq_lists
		bio_cpu_dead CPUHP_BIO_DEAD ->
			bio_alloc_cache_prune
				bio_alloc_irq_cache_splice
				__bio_alloc_cache_prune
					bio_free
		blk_softirq_cpu_dead CPUHP_BLOCK_SOFTIRQ_DEAD
			blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
				lo_complete_rq
				null_complete_rq
				nvme_pci_complete_rq (nvme_mq_ops)
					nvme_pci_unmap_rq
					nvme_complete_rq
				dm_softirq_done

	./drivers/block/nbd.c
	NBD Network block device - make block devices work over TCP

	blk_mq_init
		INIT_CSD blk_cpu_csd &__blk_mq_complete_request_remote
		open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
		cpuhp_setup_state_nocalls
		cpuhp_setup_state_multi
			__cpuhp_setup_state ->
		CPUHP_BLOCK_SOFTIRQ_DEAD &blk_softirq_cpu_dead
		CPUHP_AP_BLK_MQ_ONLINE &blk_mq_hctx_notify_offline
		CPUHP_BLK_MQ_DEAD &blk_mq_hctx_notify_dead
		blk_mq_hctx_notify_online
			clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);


	...
		blk_mq_complete_request_remote
			blk_mq_raise_softirq
				raise_softirq(BLOCK_SOFTIRQ); ->
		__blk_mq_complete_request_remote
			__raise_softirq_irqoff(BLOCK_SOFTIRQ) ->

		blk_done_softirq
			blk_complete_reqs
			this_cpu_ptr(&blk_cpu_done));


	nvme_probe
		nvme_alloc_admin_tag_set
			blk_mq_alloc_queue.cold
				blk_mq_init_allocated_queue
					blk_mq_map_swqueue

	nvme_scan_ns
		nvme_alloc_ns
			blk_mq_alloc_disk
				__blk_mq_alloc_disk->
					blk_mq_alloc_queue
						blk_mq_map_swqueue->

	__blk_mq_alloc_disk
		blk_mq_alloc_queue
			blk_mq_init_allocated_queue
				blk_mq_realloc_hw_ctxs
					blk_mq_alloc_and_init_hctx
						blk_mq_alloc_hctx
							kzalloc_node
							blk_mq_hw_ctx
								request_queue
							zalloc_cpumask_var_node(&hctx->cpumask, gfp, node)
								alloc_cpumask_var_node ->
						blk_mq_init_hctx
							init_hctx -> nvme_init_hctx
							cpuhp_state_add_instance_nocalls
							CPUHP_AP_BLK_MQ_ONLINE cpuhp_online
							CPUHP_BLK_MQ_DEAD cpuhp_dead
							blk_mq_init_request
								init_request
									nvme_pci_init_request
			blk_mq_init_cpu_queues
				blk_mq_map_queue_type
			blk_mq_map_swqueue ->

	blk_mq_end_request
		blk_update_request
			trace_block_rq_complete
				block_rq_complete
					bpf_map_lookup_elem
		__blk_mq_end_request
			blk_account_io_done
				trace_block_io_done

	blk_mq_sched_insert_requests
		blk_mq_insert_requests
			trace_block_rq_insert
				block_rq_insert

		blk_mq_alloc_tag_set
			mq_map
		blk_mq_free_tag_set
			mq_map
		blk_mq_hw_queue_to_node
			mq_map usage
			cpu_to_node

	nbd_start_device ...
	nvme_pci_update_nr_queues
		blk_mq_update_nr_hw_queues
			__blk_mq_update_nr_hw_queues
				blk_mq_map_swqueue mq_map usage
					request_queue
					hctx_idx = set->map[j].mq_map[i] usage
					__blk_mq_alloc_map_and_rqs hctx_idx
						blk_mq_alloc_map_and_rqs
							blk_mq_alloc_rq_map
								blk_mq_get_hctx_node
									hctx_idx_to_type
									blk_mq_hw_queue_to_node ->
							blk_mq_alloc_rqs
							blk_mq_free_rq_map
					Ming Lei:
					queue_for_each_hw_ctx
						hctx_table
					cpu_is_isolated
					cpumask_clear_cpu
					blk_mq_hw_ctx
						cpumask
					cpumask_set_cpu(i, hctx->cpumask);
					blk_mq_map_queue_type ->

		map_queues
	?

			pci_irq_vector(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector)
		IRQF_NOBALANCING
			desc = irq_to_desc(pci_irq_vector(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector));
			irq_settings_set_no_balancing(desc);
			irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
		queue_request_irq
		blk_mq_pci_map_queues

		blk_mq_end_request_batch
			blk_complete_request
			blk_mq_flush_tag_batch
				blk_mq_sub_active_requests
				blk_mq_put_tags
					sbitmap_queue_clear_batch
						sbitmap_queue_wake_up

=== storage busses

==== nvme



		nvme_init
			nvme_driver
				nvme_probe
					nvme_pci_alloc_dev
						nvme_max_io_queues
					nvme_pci_enable
						...
							nvme_init_queue
						#define PCI_IRQ_ALL_TYPES	(PCI_IRQ_INTX | PCI_IRQ_MSI | PCI_IRQ_MSIX)
						pci_alloc_irq_vectors->
						nvme_pci_configure_admin_queue
							nvme_alloc_queue
							nvme_init_queue
							queue_request_irq
					nvme_setup_io_queues
						nvme_setup_irqs (managed)
							irq_affinity
							PCI_IRQ_AFFINITY
							nvme_calc_irq_sets
								io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
								irq_affinity
								affd->set_size
							pci_alloc_irq_vectors_affinity ->
								masks[i].is_managed = 1
						adminq:
						queue_request_irq
							cq_vector
							nvme_irq_check
							nvme_irq
								nvme_poll_cq
								nvme_pci_complete_batch
									nvme_complete_batch
										nvme_complete_batch_req
											trace_nvme_complete_rq
										blk_mq_end_request_batch
							pci_request_irq ->
								pci_irq_vector(dev, nr)
								request_threaded_irq ->
						nvme_create_io_queues
							nvme_alloc_queue
							max = min(dev->max_qid, dev->ctrl.queue_count - 0);
							nvme_create_queue
								nvme_init_queue
									dev->online_queues++;
								cq_vector
								queue_request_irq ->
	nvme_alloc_ns
		device_add_disk
	nvme_loop_init_module
		nvme_loop_transport - host fabrics
			nvme_loop_create_ctrl
				nvme_loop_create_io_queues
					nvme_loop_mq_ops
						nvme_loop_init_request
							nvme_loop_init_iod
							nvme_loop_queue_rq

				nvme_init_ctrl
		nvme_loop_ops - target
			nvme_loop_add_port
				nvme_loop_ports
			nvme_loop_queue_response
				nvmet_req

	really_probe
		pci_device_probe
			local_pci_probe
				nvme_probe
					blk_mq_tag_set
					tagset
					nvme_alloc_io_tag_set
						blk_mq_alloc_tag_set
							mq_map = kcalloc_node
							blk_mq_update_queue_map
								map_queues
								nvme_pci_map_queues->
							blk_mq_alloc_set_map_and_rqs
								__blk_mq_alloc_rq_maps
				nvme_pci_enable->

	nvme_probe
		nvme_mq_ops
			queue_rq -> nvme_queue_rq,
				submit_bio → blk_mq_submit_bio → blk_mq_dispatch_rq_list → .queue_rq (= nvme_queue_rq)

				nvme_prep_rq
				nvme_write_sq_db
			map_queues
			nvme_pci_map_queues (nvme-pci: allow unmanaged interrupts)
				map->nr_queues = dev->io_queues[i];
				blk_mq_queue_map
				blk_mq_tag_set
				offset = queue_irq_offset(dev); (0,1)
				blk_mq_pci_map_queues Ming Lei
					blk_mq_queue_map
					pci_irq_get_affinity
						msi_desc
						pci_irq_vector
							msi_get_virq
								MSI_DEFAULT_DOMAIN
								msi_domain_get_virq
									..msi_device_data
										msi_dev_domain
									msi_desc
						irq_get_msi_desc ->
						msi_desc
							irq_affinity_desc
								mask
						&desc->affinity[idx].mask;
					mq_map set, CPU ID to hardware queue index map.
				HCTX_TYPE_POLLL:
				blk_mq_map_queues
					group_cpus_evenly
						alloc_node_to_cpumask
							nr_node_ids
						build_node_to_cpumask
							cpu_to_node
								__cpu_to_node
									x86_cpu_to_node_map
							cpumask_set_cpu
						__group_cpus_evenly
							get_nodes_in_cpumask
								cpumask_intersects
							alloc_nodes_groups
							grp_spread_init_one
								topology_sibling_cpumask
					mq_map

		    common_interrupt
		    	call_irq_handler
				handle_irq
					generic_handle_irq_desc ->

			x __common_interrupt
			...
				handle_edge_irq
					handle_irq_event
						__handle_irq_event_percpu
							action->handler
							nvme_irq ->

==== SCSI
		init_scsi drivers/scsi/scsi.c
			scsi_init_queue
				kmem_cache_create  ▻
				mempool_create_slab_pool
					mempool_create  ▻
			scsi_init_procfs
				proc_scsi_operations
			scsi_init_devinfo
				proc_scsi_devinfo_read
				proc_scsi_devinfo_write
			scsi_init_hosts
				class_register
					&shost_class
			scsi_init_sysctl
				scsi_table_header = register_sysctl_table(scsi_root_table, 1);
			scsi_sysfs_register
				bus_register scsi_bus_type
				class_register sdev_class
			scsi_netlink_init
				netlink_register_notifier(&scsi_netlink_notifier);
				netlink_kernel_create(NETLINK_SCSITRANSPORT,
					__netlink_create  ▻
		scsi_driver
			drv->bus = &scsi_bus_type
				bus_type  ▻
			device_driver
		#Scsi_Device_Template

		#Scsi_Device
		scsi_device
			request_queue
			scsi_cmnd  ▻

		SCSI Generic (sg) Driver

		sd.c
			scsi_disk Scsi_Disk
			sd_ioctl
				blk_ioctl
				scsi_ioctl
					...
					sg_write
						SG_DXFER_FROM_DEV
						sg_common_write
		init_sd
			register_blkdev
				major_names

			scsi_register_driver
				driver_register  ▻
			sd_template scsi_driver
				sd_probe
					sd_probe drivers/scsi/sd.c
						alloc_disk
						add_disk  ▻
						sd_fops
						sd_open
						sd_probe_async
							"Attached SCSI %sdisk"
							sd_printk
								sdev_printk  ▻
							sd_revalidate_disk
								limits.max_sectors
								blk_queue_max_hw_sectors
									max_hw_sectors
								sd_read_block_characteristics
									scsi_get_vpd_page
									queue_flag_set_unlocked QUEUE_FLAG_NONROT

		sd.h sd.c
		scsi.c
		scsi_ioctl.c
		scsicam.c scsi_proc.c scsi_error.c scsi_queue.c scsi_lib.c scsi_merge.c scsi_dma.c scsi_scan.c scsi_syms.c
		constants.h constants.c
		hosts.h hosts.c
		scsi_obsolete.h scsi_obsolete.c
		scsi_debug.h  scsi_debug.c scsi_module.c

		proc
			cat /proc/scsi/scsi
			scsi_proc_info

			proc_scsi_gen_write
				scan_scsis

			echo "scsi dump 0" > /proc/scsi/scsi
			scsi_dump_status

		headres

		include/scsi/scsi.h _LINUX_SCSI_H
			INQUIRY READ_CAPACITY
		include/scsi/scsi_ioctl.h _SCSI_IOCTL_H
		drivers/scsi/scsi.h _SCSI_H

		data

		scsi_cmnd Scsi_Cmnd
			use_sg
			cmnd
			request_buffer

		scsi_request Scsi_Request
			scsi_allocate_request
			scsi_release_request

		scsi_ioctl
		ioctl_internal_command
		scsi_wait_req
			scsi_do_req
				scsi_insert_special_req

			?
		scsi_do_cmd
			scsi_release_command
			scsi_queue_next_request
			scsi_initialize_queue
		Scsi_Device::request_queue
			scsi_request_fn

		scsi_hostlist

		CONFIG_SCSI CONFIG_SCSI_MODULE

		SCSI low

		/usr/src/linux/drivers/scsi

		aic94xx_init
			aic94xx_pci_driver
				asd_pci_probe
					Scsi_Host
					scsi_host_alloc
					scsi_add_host

		Scsi_Host_Template

		Scsi_Host

			Scsi_Cmnd::use_sg
			Scsi_Cmnd::request_buffer

		linux/drivers/scsi
		  low

		blkdevs blk_dev_struct blk_dev

		commands:
		dd
		mount
		fdisk
		iscsi_tcp_init
			iscsi_register_transport
			iscsi_tcp_transport
				iscsi_tcp_conn_create

		rx_data
			iscsit_do_rx_data

==== ATA

		ata_init

	scsi_dispatch_cmd
		atomic_inc(&cmd->device->iorequest_cnt);
		queuecommand
			?
			ata_scsi_queuecmd
				ata_scsi_dump_cdb
					ATA_DEBUG
		scsi_done
			trace_scsi_dispatch_cmd_done(cmd);
			blk_complete_request
				__blk_complete_request
	ATA_BASE_SHT
		ata_scsi_queuecmd
			__ata_scsi_queuecmd	ibata-scsi.c
				ata_scsi_translate
					ata_scsi_qc_new

	ata_wait_ready
		ata_link
		ata_link_online
			ata_phys_link_online
		"link is slow to respond, please be patient "
		ata_link_warn
			ata_link_printk
				link->ap->print_id

	ata_std_postreset
		sata_print_link_status
			ata_phys_link_online
				if (sata_scr_read(link, SCR_STATUS, &sstatus) == 0 &&
				    ata_sstatus_online(sstatus))
					(sstatus & 0xf) == 0x3;
			"SATA link down"
			sata_scr_read:
			sata_scr_read(link, SCR_STATUS, &sstatus))
			sata_scr_read(link, SCR_CONTROL, &scontrol);
			ata1: SATA link up 1.5 Gbps (SStatus 113 SControl F310)
			ata2: SATA link up 3.0 Gbps (SStatus 123 SControl F300)

	ata_port_pbar_desc
		ata_port_desc

	scsi_host_alloc
		shost->host_no = atomic_inc_return(&scsi_host_next_hn) - 1;
		scsi_error_handler
			host_eh_scheduled
			host_failed
			eh_strategy_handler
				ata_scsi_error
					scsi_eh_flush_done_q(&ap->eh_done_q);
					scsi_eh_flush_done_q
						(++scmd->retries <= scmd->allowed))
						scsi_queue_insert(scmd, SCSI_MLQUEUE_EH_RETRY);
					sata_pmp_error_handler
						.error_handler
							mv_pmp_error_handler
								sata_pmp_error_handler
									sata_pmp_eh_recover
										ata_eh_recover  ▻
		scsi_proc_hostdir_add
			proc_mkdir

	sata_mv

	Stack:  < submit_bio  < submit_bh  < block_read_full_page  < __do_page_cache_readahead  < page_cache_sync_readahead  < generic_file_aio_read  < do_sync_read  < vfs_read  < sys_read  < ret_fast_syscall

	Stack:  < ata_sff_port_init  < ata_port_alloc  < ata_host_alloc  < ata_host_alloc_pinfo  < mv_platform_probe  < platform_drv_probe  < driver_probe_device  < __driver_attach  < bus_for_each_dev  < bus_add_driver  < driver_register  < mv_init  < do_one_initcall  < kernel_init  < kernel_thread_exit

	Stack:  < ata_sff_tf_load  < ata_sff_qc_issue  < mv_qc_issue  < ata_qc_issue  < ata_scsi_translate  < scsi_dispatch_cmd  < scsi_request_fn  < __blk_run_queue  < blk_run_queue  < scsi_run_queue  < scsi_next_command  < scsi_io_completion  < blk_done_softirq

	Stack:  < ata_sff_exec_command  < ata_sff_qc_issue  < mv_qc_issue  < ata_qc_issue  < ata_scsi_translate  < scsi_dispatch_cmd  < scsi_request_fn  < __blk_run_queue  < elv_insert  < __make_request  < generic_make_request  < submit_bio  < write_page  < md_update_sb  < md_allow_write  < md_ioctl  < __blkdev_driver_ioctl  < blkdev_ioctl  < vfs_ioctl  < do_vfs_ioctl  < sys_ioctl  < ret_fast_syscall
	Stack:  < ata_bmdma_qc_issue  < mv_qc_issue  < ata_qc_issue  < ata_scsi_translate  < scsi_dispatch_cmd  < scsi_request_fn  < __blk_run_queue  < __blk_put_request  < blk_end_bidi_request  < scsi_io_completion  < blk_done_softirq  < __do_softirq  < irq_exit  < asm_do_IRQ  < __irq_svc  < scsi_dispatch_cmd  < scsi_request_fn  < __blk_run_queue  < elv_insert  < __make_request  < generic_make_request  < submit_bio  < write_page  < md_update_sb  < md_allow_write  < md_ioctl  < __blkdev_driver_ioctl  < blkdev_ioctl  < vfs_ioctl  < do_vfs_ioctl  < sys_ioctl  < ret_fast_syscall


	ata_sff_exec_command: ata2: cmd 0xEA

	ata_sff_hsm_move
		ata_hsm_qc_complete
			ata_qc_complete

	mv_init
	mv_platform_driver

	sas_ata_init_host_and_port
		ata_sas_port_alloc
			ata_port_alloc
				ata_link_init
					ata_dev_init
				ata_sff_port_init
					INIT_DELAYED_WORK(&ap->sff_pio_task, ata_sff_pio_task);

	mv_platform_probe
		ata_host_alloc_pinfo
			ata_host_alloc
				ata_port_alloc  ▻
		&mv_port_info
			.port_ops       = &mv5_ops
				&ata_sff_port_ops
					.sff_data_xfer		= ata_sff_data_xfer,
		ata_host_activate
		mv_init_host
			mv_port_init
				CONFIG_ATA_SFF
				port->data_addr = shd_base + (sizeof(u32) * ATA_REG_DATA);
				port->error_addr = port->feature_addr = shd_base + (sizeof(u32) * ATA_REG_ERR);
			reset_hc
				mv_soc_reset_hc
					mv_soc_reset_hc_port
						mv_reset_channel  ▻
			enable_leds
				SATALED
					find /sys/ | grep "ctera:green:disk.*/brightness"

					echo 0 > /sys/devices/platform/leds-gpio/leds/ctera:green:disk1/brightness
					echo 1 > /sys/devices/platform/leds-gpio/leds/ctera:green:disk4/brightness

				mv_soc_enable_leds
				mv_soc_led_blink_enable
					SOC_LED_CTRL	    = 0x2c,
				mv_soc_led_blink_disable

		ata_host_activate(host, platform_get_irq(pdev, 0), mv_interrupt, IRQF_SHARED, &mv6_sht);
			mv_interrupt  ▻
			ata_host_start
			ata_finalize_port_ops
				port_start
					mv_port_start
						dma_pool_alloc
			devm_request_irq ->
			ata_host_register
				ata_scsi_add_hosts
					scsi_add_host
						scsi_add_host_with_dma
							scsi_setup_command_freelist
							device_add
							scsi_sysfs_add_host
							scsi_proc_host_add
				ata_port_is_dummy
					ata_dummy_port_ops

			ata_port_desc

			ahci_pci_driver
				ahci_init_one
					&ata_dummy_port_ops

			mv6_sht
				ATA_NCQ_SHT
					ATA_BASE_SHT  ▻
		piix_init
			piix_init_one

	ata_port
		struct Scsi_Host	*scsi_host; /* our co-allocated scsi host */
			shost_gendev
			shost_dev
		ata_port_operations
		ata_link
			ata_eh_context	eh_context
			ata_eh_info	eh_info
				serror
			ata_port
				ata_link*        sff_pio_task_link

	mv_interrupt
		main_irq_cause = readl(hpriv->main_irq_cause_addr);
		mv_host_intr	ata_host
			mv_port_intr	ata_port
				mv_err_intr	ata_port
					sata_scr_read(&ap->link, SCR_ERROR, &serr);
					struct ata_eh_info *ehi
					ehi->serror |= serr;
					SERR_10B_8B_ERR 1 << 19
					SERR_DISPARITY	1 << 20

					edma_err_cause == 20
					if (edma_err_cause & EDMA_ERR_SERR)

	ata_std_error_handler
		ata_port_operations
		ops->hardreset
			mv_hardreset
				mv_reset_channel
					phy_errata
						mv6_phy_errata
						mv_soc_65n_phy_errata
							PHY_MODE3		= 0x310,
				sata_link_hardreset
					sata_link_resume
		ops->softreset
			mv_softreset
				ata_sff_softreset
		ata_do_eh
			ata_eh_autopsy
				ata_eh_link_autopsy	drivers/ata/libata-eh.c
					rc = sata_scr_read(link, SCR_ERROR, &serror);
						scr_read
							mv_scr_read
								mv_scr_offset
								mv_ap_base
								readl
					ehc->i.serror |= serror;
					ata_eh_analyze_serror
				ata_eh_analyze_ncq_error
					qc->err_mask |= AC_ERR_DEV | AC_ERR_NCQ;
				ata_eh_done
					ata_eh_clear_action

			ata_eh_report
				ata_eh_report < sata_pmp_error_handler < ata_scsi_error < scsi_error_handler < kthread < kernel_thread_exit
				ata_eh_link_report	drivers/ata/libata-eh.c
					ata_eh_context
						ata_eh_info
							serror
					"exception Emask 0x100 SAct 0x1 SErr 0x400000 action 0x6 frozen"
					ATA_PFLAG_FROZEN  ▻ "frozen" - ignore
					ata_get_cmd_descript	drivers/ata/libata-eh.c
					"failed command: WRITE FPDMA QUEUED"
						FPDMA = First Party DMA
					ata_err_string
						err_mask
						AC_ERR_ATA_BUS          = (1 << 4), /* ATA bus error */
						AC_ERR_ATA_BUS
						Emask 0x410 (ATA bus error)
					"SError:"
						SERR_DISPARITY Dispar
						SErr 0x4000000  ▻ SERR_DEV_XCHG DevExch "device exchanged"
					SErr - ehc->i.serror
					ata_queued_cmd *qc
					qc->err_mask & AC_ERR_NCQ ? " <F>" : ""
						ncq, NCQ = SATA Native Command Queueing
			ata_eh_recover
				ata_eh_reset
					reset = hardreset;
					"hard resetting link"
					ata_eh_about_to_do
						ata_eh_clear_action
					ata_do_reset
						ata_for_each_dev
			ata_eh_finish
				ata_eh_qc_retry
					scmd->retries--;
					__ata_eh_qc_complete
						scsi_eh_finish_cmd(scmd, &ap->eh_done_q);
							list_move_tail(&scmd->eh_entry, done_q);
				ata_eh_qc_complete
					scmd->retries = scmd->allowed;
					__ata_eh_qc_complete

	ata_scsi_error ./drivers/ata/libata-eh.c:787
		--ap->eh_tries)
		"EH pending after %d tries, giving up\n",

		ATA_EH_MAX_TRIES
		ATA_LINK_RESUME_TRIES ?
	ATA_DEBUG

	ata_qc_issue
		ata_sg_setup
			DPRINTK

	__irq_svc	asm_do_IRQ	handle_level_irq	handle_IRQ_event	mv_ial_lib_int_handler	mvSataInterruptServiceRoutine
	libata-core.c
	e
	eeesync_port_probe
			ata_bus_probe
				ata_dev_configure
					ata_device
					dev->horkage |= ata_dev_blacklisted(dev);
					ata_do_link_spd_horkage
						if (dev->horkage & ATA_HORKAGE_1_5_GBPS)
							target = 1;
						target_limit = (1 << target) - 1;
			ata_port_schedule_eh
		sata_link_hardreset
			sata_set_spd
				__sata_set_spd_needed
					limit = link->sata_spd_limit;
					sata_spd_limit
				sata_scr_write(link, SCR_CONTROL, scontrol)
					mv_scr_write sata_mv.c <- scr_write
						writelfl

	mv_ial_init
		driver_template.module

	echo /dev/sd{a,b}|xargs -n1 smartctl -l selftest

	CONFIG_SATA_MV
	./drivers/ata/sata_mv.c
	mv_platform_driver-> mv_platform_probe  ▻

==== SATA
		ahci_init
			ahci_pci_driver
				ahci_init_one
					pci_enable_device
					pci_request_regions  ▻
					pci_iomap
					ahci_host_init
					ata_device_add
						ata_host_init

=== Other storage

	piix_init	PCI IDE/ISA Xcelerator (Intel IC)
		piix_pci_driver
	PIIX	PCI IDE ISA Xcelerator, aka Intel 82371
	ICH	I/O Controller Hub

	fs/dcache.c  directory-entry cache (dcache)
		sys_getcwd
			current->fs_struct->pwd
			__d_path
	usb_stor_init
		usb_storage_driver
			storage_wrobe
				scsi_host_alloc(&usb_stor_host_template, sizeof(*us));
				usb_stor_host_template

	fs/buffer.c:

		touch_buffer
		mark_buffer_dirty

	sys_dup		fs/fcntl.c
		fget	lkm2
			fd > file
			task_struct  ▻ files_struct  ▻ fdtable  ▻ file (f_dentry)
			current : task_struct
				files : files_struct
					fdt : fdtable
						fd : file
			file
				file_operations    *f_op
			fcheck_files
				files_fdtable
				rcu_dereference
			atomic_long_inc_not_zero
		get_unused_fd
			alloc_fd
				files_fdtable
				find_next_zero_bit
				expand_files
					files_fdtable
					expand_fdtable
						files_fdtable
						copy_fdtable
		fd_install  ▻
		fput
			file_ref_put
				atomic_long_dec_return
	sys_dup2
		sys_dup3
			expand_files
			files_fdtable
			rcu_assign_pointer
			FD_SET(newfd, fdt->open_fds);
	sys_fcntl		fs/fcntl.c
		fget  ▻
		security_file_fcntl
			security_ops->file_fcntl
		do_fcntl
			F_DUPFD:
				alloc_fd
				get_file(filp);
				fd_install(err, filp);
			set_close_on_exec
			setfl
			fcntl_getlk
			fcntl_dirnotify
		fput
	sys_sysfs		fs/filesystems.c
		file_systems : file_system_type lkm2
		see /proc/filesystems
		fs_index
		fs_name
		fs_maxindex
	get_filesystem_list
		file_systems

	sys_ioprio_get		fs/ioprio.c
		get_task_ioprio
		ioprio_best
	sys_ioprio_set		fs/ioprio.c
		set_task_ioprio
			task->io_context->ioprio
	sys_flock		fs/locks.c lkm2
		flock_make_lock
		filp->f_op->flock
			nfs_flock
			fuse_file_flock
		flock_lock_file_wait
			flock_lock_file
				locks_insert_block
				locks_insert_lock
			wait_event_interruptible  ▻
	fs/namei.c
		sys_link  ▻
		sys_mkdir	lkm2
			sys_mkdirat
				vfs_mkdir
					dir->i_op->mkdir
						nfs_mkdir
						ext3_mkdir
		sys_mknod  ▻
		sys_rename
			sys_renameat
				vfs_rename
					vfs_rename_dir
						old_dir->i_op->rename
		sys_rmdir
			do_rmdir
				vfs_rmdir
		sys_symlink
			sys_symlinkat
				vfs_symlink
		sys_unlink  ▻
	fs/namespace.c
		sys_mount
			copy_mount_string
				strndup_user
					memdup_user
						copy_from_user
			copy_mount_options
				x exact_copy_from_user
					access_ok
						__range_ok
			do_mount
				do_new_mount
					x do_kern_mount
						file_system_type
						get_fs_type
							find_filesystem
							request_module
						vfs_kern_mount
							alloc_vfsmnt
							type->get_sb
							mount_fs
					do_new_mount_fc
						vfs_create_mount
						do_add_mount
							graft_tree
								attach_recursive_mnt
									attach_mnt
										__attach_mnt

		....
			squashfs_mount
				mount_bdev
					blkdev_get_by_path
					set_bdev_super
					sget
					fill_super  ▻ squashfs_fill_super
				squashfs_fill_super
					squashfs_read_table
						squashfs_read_data
							sb->s_fs_info
							sb_getblk
								sb->s_bdev
								__getblk_gfp
									__find_get_block
										lookup_bh_lru
											bh_lrus.bhs
									__getblk_slow
										__find_get_block  ▻
										grow_buffers
											alloc_page  ▻
							ll_rw_block
								buffer_head
								submit_bh
									submit_bh_wbc
										submit_bio
											generic_make_request
					SQUASHFS_MAGIC
		sys_pivot_root
			detach_mnt
			attach_mnt
		sys_umount
			do_umount
				umount_tree

	fs/notify/inotify/inotify_user.c
		sys_inotify_init	lkm2
			sys_inotify_init1
				inotify_fops
					inotify_read
						inotify_event
						get_one_event
							fsnotify_peek_notify_event
						copy_event_to_user
				inotify_new_group
				alloc_file  ▻
		sys_inotify_add_watch
			inotify_find_inode
				user_path_at
					pathname > path
					do_path_lookup
				inode_permission
			inotify_update_watch
				inotify_update_existing_watch
				inotify_new_watch
		sys_inotify_rm_watch
	fs/open.c
		sys_access
			sys_faccessat
				user_path_at  ▻
				inode_permission
		sys_fchdir
			file = fget(fd);
			set_fs_pwd  ▻
		sys_chdir	lkm2
			ksys_chdir
				set_fs_pwd
					current task_struct  ▻fs_struct->pwd
			inode_permission	lkm2 security
			x
			user_path_dir
				user_path_at  ▻
		sys_fchmod
			notify_change  ▻
		sys_chmod	lkm2	security
			sys_fchmodat
				mnt_want_write
				notify_change	lkm2
					inode_change_ok
					inode->i_op->setattr (inode_operations)
						ext3_setattr
						ext4_setattr
					vfs_dq_transfer
					security_inode_setattr
						inode_setattr	lkm2
							selinux_inode_setattr
							inode->i_uid
					simple_setattr
						setattr_copy
					fsnotify_change
						fsnotify_parent
							__fsnotify_parent
								fsnotify
						fsnotify	lkm2
							send_to_group
								fsnotify_create_event
								group->ops->handle_event	fsnotify_ops
									inotify_handle_event
										fsnotify_add_notify_event
									dnotify_handle_event
									fanotify_handle_event	lkm2
										fsnotify_add_notify_event
									audit_watch_handle_event
									audit_tree_handle_even
		sys_lchown
			chown_common  ▻
		sys_fchownat
			chown_common  ▻
		sys_fchown
			chown_common  ▻
		sys_chown
			chown_common
				struct inode *inode = path->dentry->d_inode;

				security_path_chown
					security_ops->path_chown
				notify_change  ▻
		sys_chroot	lkm2
			security_path_chroot
			set_fs_root
				fs->root

		sys_close  ▻
		sys_creat  ▻
		sys_fallocate
			do_fallocate
				inode->i_op->fallocate
		sys_truncate
			do_sys_truncate
				do_truncate  ▻
		sys_ftruncate
			do_sys_ftruncate
				do_truncate
					notify_change
		sys_openat
			do_sys_open  ▻
		sys_open  ▻
		sys_fstatfs
			vfs_statfs_native   ▻
		sys_statfs
			user_path
				pathname > path
				user_path_at
			vfs_statfs_native
				vfs_statfs
					dentry  ▻ kstatfs
					dentry->d_sb->s_op->statfs
		sys_vhangup
			tty_vhangup_self
				tty_vhangup
					do_tty_hangup

		dentry path inode
	fs/pipe.c
		sys_pipe	lkm2
			sys_pipe2
				do_pipe_flags
					create_write_pipe
						alloc_file  ▻
					create_read_pipe
						alloc_file  ▻
					audit_fd_pair  ▻
					fd_install  ▻
	fs/quota/quota.c
		sys_quotactl
			do_quotactl
	fs/readdir.c
		sys_getdents64
			vfs_readdir
				file->f_op->readdir
		sys_getdents
			vfs_readdir  ▻
				file->f_op->readdir
					nfs_readdir
					ext3_readdir
					ext4_readdir
						ext4_rec_len_from_disk
						filldir
	fs/read_write.c
		sys_lseek
			vfs_llseek
				default_llseek
					lock_kernel
					unlock_kernel
				file->f_op->llseek
		sys_pread64
			vfs_read  ▻
		sys_preadv
			vfs_readv  ▻
		sys_pwrite64
		sys_pwritev
		sys_read  ▻
		sys_readv
			iovec lkm2
			vfs_readv
				x do_readv_writev
				import_iovec
					rw_copy_check_uvector
					iov_iter_init
				do_iter_read
					iov_iter_count
					do_iter_readv_writev
						init_sync_kiocb
						file->f_op->read_iter
							generic_file_read_iter
								mapping->a_ops->direct_IO
					x do_sync_readv_writev
						file->f_op->aio_read
						file->f_op->aio_write
					do_loop_readv_writev
						iov_iter_iovec
						iov_iter_advance
							iterate_and_advance
								iterate_iovec
							iov_length
								iov_len
		iov_iter_truncate
			i->count = count

		sys_copy_file_range fs/read_write.c
			vfs_copy_file_range
				do_copy_file_range
					generic_copy_file_range
						MAX_RW_COUNT
						do_splice_direct ->
		sys_sendfile	lkm2 fs/read_write.c
		sys_sendfile64
			do_sendfile
				do_splice_direct	lkm2 fs/splice.c
					n do_splice_direct_actor
						splice_direct_to_actor
							n do_splice_read
								copy_splice_read
									iov_iter_bvec
									read_iter
									pipe_head_buf
										pipe_buf
							x do_splice_to  ▻
								splice_read
									sock_splice_read  ▻ ./net/socket.c
					direct_splice_actor
						do_splice_from
							splice_write
								x generic_splice_sendpage fs/splice.c
									actor = pipe_to_sendpage
										sendpage
											x sock_sendpage  ▻ ./net/socket.c
									splice_from_pipe  ▻
								out->f_op->splice_write
									splice_to_socket
										sock_sendmsg ->
								default_file_splice_write
									actor = write_pipe_buf
										kernel_write
											vfs_write  ▻
									splice_from_pipe
										__splice_from_pipe

		sys_write  ▻
		sys_writev
			iovec
			vfs_writev
				iov_iter
				import_iovec
				do_iter_write
					do_iter_readv_writev
						call_read_iter
							read_iter
						call_write_iter
							write_iter
					do_loop_readv_writev
		fs/select.c
			DEFAULT_POLLMASK	POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM
			sys_poll	lkm2
				do_sys_poll
					poll_initwait
						qproc  = __pollwait
						init_poll_funcptr
					do_poll
						do_pollfd
							file->f_op->poll
						poll_schedule_timeout

						xxx poll_wait
							p->qproc
								__pollwait
									init_waitqueue_func_entry
									add_wait_queue
										__add_wait_queue
			sys_ppoll
				poll_select_set_timeout
				do_sys_poll  ▻
			sys_pselect6
				do_pselect
					core_sys_select  ▻
			sys_select   ▻ lkm2

		fs/signalfd.c
			sys_signalfd
				sys_signalfd4
					anon_inode_getfd  ▻
		fs/splice.c
			sys_splice	lkm2
				do_splice
					splice_pipe_to_pipe
					do_splice_to
						splice_read
							sock_splice_read  ▻
			sys_tee	lkm2
				do_tee
					ipipe_prep
						pipe_wait_readable
						pipe_wait_writable
					opipe_prep
					link_pipe
						pipe_buffer
						wakeup_pipe_readers
			sys_vmsplice
				vmsplice_to_pipe
					splice_to_pipe
				vmsplice_to_user
					__splice_from_pipe
		fs/stat.c
			sys_newfstatat
				vfs_fstatat
					user_path_at  ▻
					vfs_getattr	lkm2
						inode->i_op->getattr
							nfs_getattr
						generic_fillattr
							inode  ▻ kstat
				cp_new_stat
					kstat  ▻ stat
			sys_newfstat	lkm2
				vfs_fstat
					fget
					vfs_getattr
				cp_new_stat
			sys_newlstat
				vfs_lstat
					vfs_fstatat  ▻
				cp_new_stat
			sys_newstat
				vfs_stat
					vfs_fstatat  ▻
				cp_new_stat
			sys_readlink
				sys_readlinkat
	fs/super.c
		sys_ustat
			vfs_statfs
	fs/timerfd.c
		sys_timerfd_create
			anon_inode_getfd  ▻
		sys_timerfd_gettime
		sys_timerfd_settime ->
	fs/utimes.c
		sys_utimes
			sys_futimesat  ▻
		sys_futimesat
			do_utimes
				utimes_common
		sys_utime
			do_utimes
		sys_utimensat
			do_utimes
	fs/xattr.c
		sys_setxattr
			user_path  ▻
			setxattr
				vfs_setxattr
					xattr_permission
					__vfs_setxattr_noperm
						inode->i_op->setxattr
		sys_listxattr
			listxattr
				vfs_listxattr
		sys_getxattr
			getxattr
				vfs_getxattr
		sys_removexattr
			removexattr
		sys_fgetxattr
			audit_inode
				__audit_inode
		sys_flistxattr
		sys_fremovexattr
		sys_fsetxattr
		sys_lgetxattr
		sys_lsetxattr
			user_lpath
				user_path_at  ▻
			mnt_want_write
		sys_llistxattr
		sys_lremovexattr
	blkdev_ioctl
	fio
		fio_options
		engines
			psync
				fio_psyncio_queue
					pread
					pwrite
					do_io_u_sync

	bio_endio
		trace_block_bio_complete
			block_bio_complete

	__trace_req_completion
		bpf_ktime_get_ns

	./biosnoop.bt
		kprobe:blk_account_io_start
		kprobe:blk_account_io_done

	blk_execute_rq_nowait
		blk_account_io_start
			trace_block_io_start

	bioset_init
		cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
			__cpuhp_state_add_instance
				cpus_read_lock();
				__cpuhp_state_add_instance_cpuslocked
					mutex_lock(&cpuhp_state_mutex);
					hlist_add_head
	bio_alloc_cache_destroy
		cpuhp_state_remove_instance_nocalls
			__cpuhp_state_remove_instance
				cpus_read_lock
				mutex_lock(&cpuhp_state_mutex);

	init_bio
		cpuhp_setup_state_multi
		CPUHP_BIO_DEAD &bio_cpu_dead

	./bcc/tools/xfsslower.py
		events.perf_submit
	xfsslower
	    "trace_fsync_entry"
	    attach_kprobe
		bpf_attach_kprobe
		    bpf_attach_probe
			bpf_try_perf_event_open_with_probe
			    perf_event_open

	fsslower
	    bpf_program__attach_kprobe

	strace --follow-forks --attach="$(pidof etcd)" -Tt --signal='!SIGURG,SIGPIPE' --status=successful -e trace=\!futex |& grep -v --line-buffered -F '0.0'

	del_timer_sync
		try_to_del_timer_sync

	 => ftrace_ops_list_func
	 => ftrace_call
	 => ttwu_do_wakeup
	 => try_to_wake_up
	 => hrtimer_wakeup
	 => __hrtimer_run_queues
	 => hrtimer_interrupt
	 => smp_apic_timer_interrupt
	 => apic_timer_interrupt
	 => __x86_indirect_thunk_rax
	 => __rq_qos_track
	 => blk_mq_make_request
	 => generic_make_request
	 => submit_bio
	 => iomap_submit_ioend.isra.38
	 => xfs_vm_writepages
	 => do_writepages
	 => __filemap_fdatawrite_range
	 => file_write_and_wait_range
	 => xfs_file_fsync
	 => do_fsync
	 => __x64_sys_fdatasync
	 => do_syscall_64
	 => entry_SYSCALL_64_after_hwframe


	 => trace_event_raw_event_sched_switch
	 => __schedule
	 => schedule
	 => schedule_timeout
	 => io_schedule_timeout
	 => balance_dirty_pages
	 => balance_dirty_pages_ratelimited
	 => iomap_write_actor
	 => iomap_apply
	 => iomap_file_buffered_write
	 => xfs_file_buffered_aio_write
	 => new_sync_write
	 => vfs_write
	 => ksys_pwrite64
	 => do_syscall_64
	 => entry_SYSCALL_64_after_hwframe

	blkcg_iolatency_ops
		blkcg_iolatency_throttle
			__blkcg_iolatency_throttle
				rq_qos_wait ^
}

== Networking

	netdev_priv
	kfree_skb
		SKB_DROP_REASON_NOT_SPECIFIED
		kfree_skb_reason
			sk_skb_reason_drop
				__kfree_skb
					skb_release_all
						skb_release_data
					kfree_skbmem
						kmem_cache_free

	mtk_star_probe
		net_device
		devm_alloc_etherdev
		devm_platform_ioremap_resource
		devm_regmap_init_mmio
		devm_clk_bulk_get
		devm_register_netdev

	inet_init  ▻
	copy_process <-
		copy_namespaces
			create_new_namespaces
				copy_net_ns
					mutex_lock_killable(&net_mutex);

	/proc/net

	bt_init
		bt_sysfs_init
		sock_register(&bt_sock_family_ops);
		bt_sock_family_ops
			bt_sock_create,
		hci_sock_init
			proto_register(&hci_sk_proto, 0);
			bt_sock_register(BTPROTO_HCI, &hci_sock_family_ops);

	ipw2100_pci_init_one
		ipw2100_open

	usbnet_probe
		usbnet_netdev_ops
			usbpn_open
				usb_set_interface
				rx_submit
				netif_wake_queue
			usbnet_start_xmit
		register_netdev
		usbnet_ethtool_ops

		kevent
			rx_submit
				__netdev_alloc_page
				usb_submit_urb
				rx_complete
					defer_bh
					rx_submit  ▻

	cdc_init
		cdc_driver
			usbnet_probe

	ip_rt_do_proc_init
		rt_cache_seq_fops
			fib_trie_seq_show x rt_cache_seq_show

		rt_cpu_seq_show

	udp4_proc_init
		udp4_net_ops
	net_dev_init
		dev_proc_ops
			dev_seq_fops
				dev_seq_ops
					sg_proc_seq_show_dev
		dev_proc_init

=== Sockets
		sys_socketcall <- system_calls
			sys_socket
				sock_create create
					__sock_create 100 LOC
						socket sock_alloc
							new_inode
						security_socket_create
						RCU Read-Copy Update
						pf = rcu_dereference(net_families[family]);
						pf->create
							? inet_create
								inetsw
								sk_alloc
								sock_init_data
								?	.init			= tcp_v4_init_sock, (tcp_prot)
									ipv4_specific inet_connection_sock_af_ops  af = address family
										.queue_xmit	   = ip_queue_xmit,
									...
				sock_map_fd
			sys_connect connect
				sock->ops->connect
					inet_stream_connect
					inet_dgram_connect
			sys_setsockopt
				sock_setsockopt
				sock->ops->setsockopt
			sys_getsockopt
				sock_getsockopt
			sys_sendmsg  ▻ sock_sendmsg
			sys_recvmsg  ▻ sock_recvmsg  ▻

			sys_accept
				sys_accept4
			sys_bind
			sys_listen
				inet_listen inet_stream_ops
					inet_csk_listen_start
						sk->sk_prot->get_port

			sockfd_lookup sock_map_fd


	??
		  - sock_sendpage -> splice_to_socket
		  - tcp_sendpage -> ?
		  - udp_sendpage -> ?

		

		socket_file_ops	file_operations
			splice_write -> splice_to_socket,
			sock_ioctl
			sock_aio_read  ▻
	??
			af_unix_init
				sock_register unix_family_ops
			x sock_sendpage	sendpage
				kernel_sendpage
					proto sendpage
						inet_sendpage
							tcp_sendpage  ▻
							udp_sendpage
								ip_append_page

			generic_splice_sendpage,	.splice_write
			sock_splice_read	lkm2 .splice_read
				splice_read
					tcp_splice_read, lkm2
						__tcp_splice_read
							tcp_read_sock
								tcp_recv_skb
			sock_poll	poll

	netdev_alloc_skb
		__netdev_alloc_skb
			__alloc_skb
			skb_reserve
	dev_alloc_skb
		__dev_alloc_skb
			alloc_skb
				__alloc_skb
					kmem_cache_alloc_node
						kmem_cache_alloc  ▻
			skb_reserve
	netif_rx

=== eth
			input output samples
			sk_buff

		=== eth_data
			net_device linux/netdevice.h
				alloc_netdev
					kzalloc
			hard_start_xmit
			dev_base  // All devices
				alloc_etherdev linux/etherdevice.h net/ethernet/eth.c
					alloc_etherdev_mq
						alloc_netdev_mq	lkm2
					ether_setup
						dev->hard_header	= eth_header;
					alloc_netdev
				register_netdev  lkm2
					dev_alloc_name
					register_netdevice
						dev->init
						netdev_register_sysfs
							class_device_add
			ethhdr
				h_dest
				h_source
				h_proto
		ksoftirqd
	dm9000_init
		platform_driver_register(&dm9000_driver);
			dm9000_probe
				dm9000_poll_work

			netpoll_poll
				ndo_poll_controller
					dm9000_poll_controller
						dm9000_interrupt  ▻

			dm9000_timeout
				netif_stop_queue
					set_bit(__LINK_STATE_XOFF, &dev->state);
				dm9000_reset
				dm9000_init_dm9000
				netif_wake_queue
					__netif_schedule
			dm9000_stop
				del_timer
				netif_stop_queue
				netif_carrier_off
			dm9000_shutdown
			dm9000_open
				dm9000_reset
					dm9000_init_dm9000
					dm9000_hash_table
					init_timer
				mii_check_media
				netif_start_queue
					clear_bit(__LINK_STATE_XOFF, &dev->state);
		dev_queue_xmit
			hard_start_xmit
				dm9000_start_xmit
					netif_stop_queue

		dev_watchdog
			netif_queue_stopped
			dm9000_timeout

		dm9000_interrupt
			dm9000_rx
				dev_alloc_skb
				netif_rx
					input_pkt_queue
					__skb_queue_tail
					process_backlog
							netif_receive_skb

		dm9000_timer
			mii_check_media
				netif_carrier_ok
				mii_link_ok
					mdio_read
				netif_carrier_on
				netif_carrier_off

		 == eth_output output
			dev_queue_xmit  ▻ hard_start_xmit

		 == eth_input input
		intrerrupt (old)
			netif_rx
				cpu_raise_softirq NET_RX_SOFTIRQ
					net_rx_action
			 ....
		netif_receive_skb ??
			__netif_receive_skb
					ip_rcv  ▻ tcp socket_input
		== eth_samples samples
			pci_driver e100_driver
				e100_probe
					net_device *netdev = alloc_etherdev
						e100_open
						e100_close
						e100_xmit_frame
						e100_get_stats
						e100_set_multicast_list
						e100_set_mac_address
						e100_change_mtu
						e100_do_ioctl
						e100_ethtool_ops
						e100_tx_timeout
						e100_poll
						e100_netpoll

			e1000
				e1000_init_module e1000_exit_module
				e1000_driver
					e1000_probe
						e1000_netdev_ops
							e1000_open

				netdev
					hard_start_xmit
						.ndo_start_xmit
						e1000_xmit_frame sk_buff lkm2
							e1000_tso
								skb->data

							e1000_tx_queue
								writel
				e1000_up
					e1000_intr	lkm2
						e1000_clean_rx_irq (2.4-18-3) // Send received data up the network stack
							skb_put
							e1000_rx_checksum
							netif_rx net/core/dev.c
					... e1000_receive_skb
						netif_receive_skb  ▻

				e1000_poll (2.4-20)
					rx_ring
					e1000_process_intr
					netif_receive_skb  ▻

			e100_found1 sis900_pci_driver

		ieee80211_init

		ieee80211_alloc_hw
			mac80211_config_ops
				ieee80211_add_iface,
					ieee80211_if_add
						alloc_netdev_mq  ▻
						ieee80211_if_setup,
							ether_setup
						ieee80211_dataif_ops
						ieee80211_open,
							drv_add_interface
		ieee80211_register_hw
			ieee80211_if_add

		x eepro100_init_module
			eepro100_driver
				eepro100_init_one
					speedo_found1
						speedo_open
							speedo_interrupt
								speedo_rx
								speedo_tx_buffer_gc

			ethtool_ops
				speedo_get_link get_link
					mii_link_ok
						mdio_read BMSR_LSTATUS
							iowrite32
			dev_ethtool
				ETHTOOL_GLINK ethtool_get_link
					if (!dev->ethtool_ops->get_link)
			ethtool_op_get_link
				netif_carrier_ok
					__LINK_STATE_NOCARRIER

			netif_carrier_on	lkm2
			netif_carrier_off

			bond_check_dev_link
				netif_carrier_ok
				if_mii BMSR_LSTATUS
				slave_dev->ethtool_ops->get_link(slave_dev);

		net\ipv4\icmp.c

		fib (Forwarding Information Base)

		net\ipv4\ls
		net\ipv4\af_inet.c
		include\net

		tcp4_proc_init
			register_pernet_subsys(&tcp4_net_ops);
			tcp4_net_ops
				tcp4_seq_show

		inet_family_ops net_proto_family
				inet_create  ▻
		inet_init
			proto_register
				proto_list
			tcp_prot  ▻
			udp_prot
				ip4_datagram_connect
				udp_sendmsg
					sock_tx_timestamp
					ip_append_data
						__skb_queue_tail(&sk->sk_write_queue, skb);
					ip_route_output_flow
					? ip_cmsg_send
					udp_push_pending_frames
						ip_push_pending_frames
							ip_local_out
								dst_output
									skb->dst->output
										? ip_output
					ip_rt_put
				udp_recvmsg
					__skb_recv_datagram
						sock_rcvtimeo
						skb_peek
					? ip_cmsg_recv
					wait_for_packet
					? dst_input
						? ip_local_deliver  ▻
			raw_prot
				ip4_datagram_connect,
			sock_register inet_family_ops
				net_families
			icmp_protocol
			udp_protocol
				.handler =	udp_rcv,
					__udp4_lib_rcv
						udp_queue_rcv_skb
			tcp_protocol
				tcp_v4_rcv
			igmp_protocol
			inetsw_array
			inet_register_protosw
				inetsw

			arp_init
			ip_init
			tcp_v4_init(&inet_family_ops);
				if (register_pernet_subsys(&tcp_sk_ops))
				inet_csk_ctl_sock_create
					sock_create_kern
				tcp_socket
			tcp_init
			udplite4_register
			icmp_init
			#if defined(CONFIG_IP_MROUTE)
				ip_mr_init();
			#endif
			dev_add_pack(&ip_packet_type);
				packet_type ip_packet_type
					.type = __constant_htons(ETH_P_IP),
					.func = ip_rcv,
					.gso_send_check = inet_gso_send_check,
					.gso_segment = inet_gso_segment,

		include\linux\net.h, BSD
		SYS_SOCKET - syscalls list
		socket
			socket_state		state;
			unsigned long		flags;
			const struct proto_ops	*ops;
			struct fasync_struct	*fasync_list;
			struct file		*file;
			struct sock		*sk;
			wait_queue_head_t	wait;
			short			type;
		proto_ops
		net_proto_family
		net_proto
		sock	(8)	TCP/IP

		net\ipv4\arp.c

		include\linux\netdevice.h
		net_device

		include\linux\inetdevice.h
		in_device
		net\ipv4\devinet.c
		drivers/net

		18. Networking
		      Main Networking Data Structures
		      System Calls Related to Networking
		      Sending Packets to the Network Card
		      Receiving Packets from the Network Card

		Documentation\networking
		include\net
		net
		net\ipv4

		eth

		ifconfig ... up
			dev_ioctl
				dev_ifsioc
					SIOCSIFFLAGS:
						dev_change_flags
							__dev_change_flags
								__dev_open
									ops->ndo_open
								__dev_close
									__dev_close_many
										ops->ndo_stop
											ieee80211_stop  ▻
											zd1201_net_stop
											lbs_eth_stop
											airo_close
												netif_stop_queue  ▻
										dev->flags &= ~IFF_UP;
							rtmsg_ifinfo
							__dev_notify_flags
								call_netdevice_notifiers(NETDEV_DOWN, dev);
									raw_notifier_call_chain
										__raw_notifier_call_chain
											notifier_call_chain
												notifier_call CB
	ieee80211_stop
		ieee80211_do_stop
			....
			ieee80211_stop_device
				drv_stop
					ieee80211_ops:stop
						ath9k_stop
						ath9k_htc_stop
							htc_stop
								ath9k_htc_hif::stop
						iwlagn_mac_stop
							iwl_down
								__iwl_down
									iwl_clear_driver_stations
						iwl3945_mac_stop
						zd_op_stop

	.....
		ieee80211_mgd_assoc
			ieee80211_add_work
			ieee80211_assoc_done
			ieee80211_assoc_success
			ieee80211_set_associated

		ieee80211_reconfig
			case NL80211_IFTYPE_STATION:
				changed |= BSS_CHANGED_ASSOC;
				mutex_lock(&sdata->u.mgd.mtx);
				ieee80211_bss_info_change_notify(sdata, changed);

		iwlagn_mac_start	ieee80211_ops::start
			__iwl_up
				iwl_prepare_card_hw
				iwlagn_hw_nic_init
				iwl_nic_start

		netdevice->flags

			IFF_UP

			IFF_RUNNING

=== IP
			include/linux/ip.h _LINUX_IP_H
			iphdr

==== ip_output ip outout

		ip_queue_xmit  # sk_buff (net/ipv4/ip_output.c)
			ip_route_output  ▻
			rt = (struct rtable *)__sk_dst_check(sk, 0);
			NF_HOOK rt->u.dst.dev
				nf_hook_slow # skb net_device okfn = ip_queue_xmit2
					nf_queue
						status = queue_handler[pf].outfn(skb, info, queue_handler[pf].data);

				???
			ip_queue_xmit2 # skb, net/ipv4/ip_output.c
				skb->dst->output(skb)
					ip_output ? # sk_buff
						ip_finish_output
							ip_fragment
							ip_finish_output2
								dst = skb->dst;
								dst->hh;
								hh->hh_output(skb); (hh_cache hardware header)
									? dev_queue_xmit (net/core/dev.c)
										...qdisc_restart
											dev->hard_start_xmit
											dev_queue_xmit_nit
												-> eth_output
								dst->neighbour->output(skb);
									neigh_resolve_output
										dst_entry *dst = skb->dst;
			 							neighbour *neigh = dst->neighbour
										dev->hard_header neigh->ha
											eth_header # Create the Ethernet MAC header
										neigh->ops->queue_xmit(skb);
										# ha = hardware address, mac!
												-> eth_output

			link_rtnetlink_table rtnetlink_links ?
				neigh_add
					neigh_update - sets neigh->ha

==== ip_input

		net/ipv4/ip_input.c
		ip_rcv <- eth eth_input
			ip_rcv_finish
				ip_route_input	lkm2
				skb->dst->input
					ip_local_deliver
						NF_HOOK
							NF_HOOK_THRESH
								nf_hook_thresh
									nf_hook_slow  ▻

						ret = ipprot->handler
						 ? udp_rcv
						 ? tcp_v4_rcv (tcp_protocol , net/ipv4/tcp_ipv4.c)
							tcp_v4_do_rcv
								tcp_rcv_established
								tcp_rcv_state_process
						ip_local_deliver_finish
				dst_input

==== ip_route ip route

			eth destination selection - sk_buff.ethernet
			source interface selection - ip_route_output_slow
			include/net/route.h
				rt_key
				rtable
			ip_route_connect # rtable dst src
				ip_route_output # rtable daddr saddr
					ip_route_output_key (35 lines) # rtable rt_key
						ip_route_output_slow (300 lines)
						{ # nh = next hop
						# fib = forward information base
							main vars
								rt_key key;
									dst src
								fib_result res;
									fib_rule
								rtable *rth;
								net_device *dev_out
							specific cases:
								ip_dev_find dev_get_by_index inet_select_addr

							# searching route in routing table

							#if !CONFIG_TUX_LOOPBACK_PHYSICAL
								if (fib_lookup(&key, &res))
									tb_lookup
							#else
								if (main_table->tb_lookup(main_table, &key, &res))
									fn_hash_lookup?
							#endif

							# get device to route

							make_route:
								rth = dst_alloc(&ipv4_dst_ops);
								rt_set_nexthop(rth, &res, 0);
								err = rt_intern_hash(hash, rth, rp);
									arp_bind_neighbour
							done:
								if (free_res) fib_res_put(&res);
						}

			sock->dst_cache
			rt_intern_hash

==== ip_forwarding
		forwarding net/ipv4/ip_forward.c
			ip_forward
				ip_send
					ip_finish_output  ▻
		arp
			arp_constructor
			arp_generic_ops
			arp_send
			arp_rcv

		# linux TCP/IP API

		keys
			create route connect send output input ops
			socket INET TCP UDP IP tcp eth->
			output send, sendto, sendmsg
			input recv, recvfrom, recvmsg

		#include <sys/types.h>

		#include <sys/socket.h>

		 == inet_layers
			sys_socketcall
			socket
			INET
			sk_buff
			sock
			TCP UDP inet_stream_ops proto_ops
			IP
			eth

		O_NONBLOCK F_SETFL fcntl

		MSG_DONTWAIT  ▻ EAGAIN

		msghdr
			msg_iter : iov_iter
				iov_iter
		user_msghdr
		kvec
		iovec (linux/uio.h)
			iov_iter
				iov_offset
					iterate_and_advance
			iov_iter_init
				used in
					import_iovec
					import_single_range
					fuse_ioctl_copy_user

		== socket
			# linux/net.h _LINUX_NET_H
			net_proto_family create
				sk sock  ▻
				proto_ops ops
					sendpage
					splice_read
				socket - BSD socket
					no connect
					sock_sendmsg
					sock_ioctl
					sock_release
				sock_create
				sock_release

		 == socket_input socket input
			sock_recvmsg
				__sock_recvmsg
					tcp_recvmsg
				used by
					kernel_recvmsg
					sock_read_iter
					recvfrom
					iscsit_do_rx_data

		 == socket_output socket output
			sock_sendmsg
				inet_sendmsg
					tcp_sendmsg  ▻
			x sock_sendpage  ▻

				...
				? tcp_sendpage
					do_tcp_sendpages
						skb_fill_page_desc
						tcp_push

		 == INET PF_INET
				inet_family_ops ops
				inet_stream_ops  ▻
				inet_dgram_ops  ▻
				subprotocols
					inetsw_array
						.prot = &tcp_prot,
						.ops =  &inet_stream_ops,
						...
						.prot =       &udp_prot,
						.ops =        &inet_dgram_ops,

						.prot =       &raw_prot,
						.ops =        &inet_sockraw_ops,
					inetsw
					inet_register_protosw
				inet_opt (net/sock.h)

		 == sock
			# net/sock.h _SOCK_H  INET socket, transport layer
				sock # internal low level socket data, after sock_lock_init
					tp_pinfo
						af_tcp tcp_opt
				proto (after sk_add_backlog)
				sock_setsockopt net/core/sock.c
					x-> inet_setsockopt
				inet_sendmsg (af_inet.c)
				sock_ioctl  ▻ inet_ioctl

		 == TCP
				# when dst or route mac are detected?
				# ref: ha
				inet_stream_ops proto_ops
					.connect	   = inet_stream_connect,
					.accept		   = inet_accept,
					.sendmsg	   = inet_sendmsg,
					.recvmsg	   = sock_common_recvmsg,
					.sendpage	   = inet_sendpage,
					.splice_read	   = tcp_splice_read,
					inet_stream_connect connect
						sk->prot->connect
							tcp_v4_connect  ▻
						inet_wait_for_connect
				tcp_prot proto
					.init			= tcp_v4_init_sock,
					.connect		= tcp_v4_connect,
					.disconnect		= tcp_disconnect,
					.sendmsg		= tcp_sendmsg,
					.recvmsg		= tcp_recvmsg,
					tcp_sendmsg
					tcp_close
					tcp_v4_connect (100 lines connect) # sock sockaddr
						ip_route_connect  ▻ (route) #
						# sk_buff
						tcp_connect # sock sk_buff (100 lines)
							tcp_transmit_skb  ▻
					tcp_ioctl
					tcp_sendmsg
				net/tcp.h _TCP_H
					tcp_sendmsg (net/ipv4/tcp.c)

		 == tcp_output output
			tcp_sendmsg iov from # 220 LOC
				skb_add_data
				tcp_copy_to_page - csum_and_copy_from_user - csum_partial_copy_generic ?
				tcp_push (net/ipv4/tcp.c)
					__tcp_push_pending_frames
						tcp_write_xmit (net/ipv4/tcp_output.c)
							tcp_transmit_skb  ▻

					...???
					.. tcp_send_skb (net/ipv4/tcp_output.c)
						tcp_write_xmit (net/ipv4/tcp_output.c)
							tcp_transmit_skb # sock sk_buff, net/ipv4/tcp_output.c)
								struct tcp_opt *tp = &(sk->tp_pinfo.af_tcp);
								tp->af_specific->send_check(sk, th, skb->len, skb);
									???? tcp_v4_send_check # This routine computes an IPv4 TCP checksum
								err = tp->af_specific->queue_xmit(skb);
									?? ip_queue_xmit  ▻

		ip_build_xmit
			getfrag
		ip_queue_xmit

		 == tcp_input tcp input
			tcp_recvmsg # 300 lines, linux/net/ipv4/tcp.c
				tcp_data_wait
				skb = skb_peek(&sk->receive_queue);
				cleanup_rbuf
				tcp_data_wait
				! tcp_prequeue_process
					sk->sk_backlog_rcv
						tcp_v4_rcv  ▻
				skb_copy_datagram_iovec  ▻
				tcp_prequeue_process  ▻
				cleanup_rbuf

				tcp_recv_urg

			sock_queue_rcv_skb

			receive_queue
				writers
					tcp_data_queue
					tcp_ofo_queue
					tcp_rcv_established
				readers
					tcp_check_urg
					tcp_recv_skb
					tcp_recvmsg

			raw_prot
				raw_sendmsg
					ip_cmsg_send
					ip_build_xmit

		 == UDP
				inet_dgram_ops proto_ops
					inet_dgram_connect
					inet_sendmsg
				udp_prot proto (ops net/ipv4/udp.c, net/udp.h)
				net/udp.h
					udp_sendmsg (net/ipv4/udp.c)  ▻
			low level: eth->

		 == sk_buff
			sk_buff skb ( linux/skbuff.h _LINUX_SKBUFF_H)
				dst (dst_entry) include/net/dst.h
					neighbour (neighbour)
						ha
					hh  (hh_cache)
						hh_output
					input
					output
		alloc_ieee80211
			dev->hard_start_xmit = ieee80211_xmit;
				skb_pull
			ieee80211_device

		zd1201_probe
			zd1201_net_open
		drivers/net/wireless/zd1211rw
		usb_init linux/drivers/net/wireless/zd1211rw/zd_usb.c
			? zd_usb_enable_rx
				alloc_urb
					rx_urb_complete
						handle_rx_packet
							zd_mac_r	x_irq
								tasklet_schedule(&mac->rx_tasklet);
									do_rx  ▻
						usb_submit_urb

			usb_register
				driver
					probe
						zd_mac_init_hw
							zd_chip_enable_int
								zd_usb_enable_int
									usb_alloc_urb
									usb_fill_int_urb	int_urb_complete
									usb_submit_urb
							zd_chip_init_hw
						register_netdev
			zd_netdev_alloc
				alloc_ieee80211softmac
				zd_netdev_mac
					ieee80211softmac_priv

			do_rx
				zd_mac_rx
					ieee80211_rx
						netif_rx

			zd_mac_init
				tasklet_init(&mac->rx_tasklet, do_rx, (unsigned long)mac);
				ieee_init
					ieee->hard_start_xmit = netdev_tx;
						zd_mac_tx
							zd_usb_tx
								usb_alloc_urb
								usb_buffer_alloc
								usb_fill_bulk_urb
								tx_urb_complete
	ieee80211s_init
	net/socket.c
		sys_accept
			sys_accept4
				sockfd_lookup_light
				sock->ops->accept
					sock_from_file
						socket_file_ops
				sock_alloc_fd
		sys_bind  ▻
		sys_connect  ▻
		sys_getpeername
			sock->ops->getname
		sys_getsockname
			sock->ops->getname
		sys_getsockopt  ▻
		sys_listen  ▻
		sys_recvfrom
			sock_recvmsg  ▻
		sys_recvmmsg
			__sys_recvmmsg
		sys_recvmsg  ▻
			sock_recvmsg  ▻
		sys_sendmsg  ▻
		sys_sendto
			sock_sendmsg  ▻
		sys_setsockopt  ▻
		sys_shutdown
			sock->ops->shutdown
		sys_socket  ▻
		sys_socketpair
			sock_create
			sock1->ops->socketpair
			sock_alloc_file
			audit_fd_pair
				__audit_fd_pair
					current->audit_context
			fd_install  ▻
	netlink_kernel_create
		sock_create_lite
		_netlink_create

	rtl8139_open
	rtl8139_interrupt
	rtl8169_open

	iwl
		iwl3945_hw_ops
			iwl3945_mac_stop
				iwl3945_down
					__iwl3945_down

			iwl_bss_info_changed
				iwl_set_no_assoc
					priv->assoc_id = 0;
					priv->staging_rxon.filter_flags &= ~RXON_FILTER_ASSOC_MSK;

	mac80211_config_ops ...

	MLME	Media Access Control (MAC) Sublayer Management Entity
	nl80211 is the new 802.11 netlink interface public header.
	SME	station management entity
		Together with cfg80211 it is intended to replace Wireless-Extensions.
	SSID	Service set identifier
	BSS	single access point (AP) together with all associated stations (STAs)

	nl80211_disassociate	NL80211_CMD_DISASSOCIATE	nl80211_ops
	nl80211_send_deauth
		nl80211_send_mlme_event(rdev, netdev, buf, len, NL80211_CMD_DEAUTHENTICATE, gfp);

	ieee80211_deauth	cfg80211_deauth_request
		ieee80211_mgd_deauth
			ieee80211_set_disassoc
			ieee80211_send_deauth_disassoc  ▻

	cfg80211_mlme_disassoc
	__cfg80211_mlme_disassoc
		disassoc = ieee80211_disassoc
		??
		ieee80211_disassoc
			ieee80211_mgd_disassoc
				ieee80211_set_disassoc
					ieee80211_hw_config
					ieee80211_bss_info_change_notify
				ieee80211_send_deauth_disassoc
					cfg80211_send_deauth
						__cfg80211_send_deauth  ▻
					__cfg80211_send_deauth
						nl80211_send_deauth  ▻
					__cfg80211_send_disassoc
						cfg80211_sme_disassoc
							__cfg80211_mlme_deauth
						nl80211_send_disassoc
							NL80211_CMD_DISASSOCIATE
							nl80211_send_mlme_event
								nl80211hdr_put
									genlmsg_put
								genlmsg_multicast_netns
									nlmsg_multicast
										netlink_broadcast  ▻
					ieee80211_tx_skb
						ieee80211_xmit
							ieee80211_tx
								__ieee80211_tx
									drv_tx
										local->ops->tx

	fragmentation
		__pskb_pull_tail
			pskb_pull
		dev_forward_change
		addrconf_ifdown

		udp_gro_receive
			NETIF_F_GRO_UDP_FWD

		veth_skb_is_eligible_for_gro
			NETIF_F_GRO_UDP_FWD

	inet_gro_receive
		gro_receive
			geneve_gro_receive

	ipip_init
		ipip_init_net
			ipip_tunnel_setup

call_function_single_interrupt
irq_exit
irq_exit_rcu
__do_softirq
net_rx_action
__napi_poll
gro_cell_poll
napi_gro_receive
netif_receive_skb_internal
? inet_gro_receive
__netif_receive_skb_core
? ip6_protocol_deliver_rcu
br_handle_frame [bridge]
br_handle_frame_finish [bridge]
__br_forward [bridge]
? fdb_find_rcu [bridge]
br_forward_finish [bridge]
br_dev_queue_push_xmit [bridge]
__dev_queue_xmit
sch_direct_xmit
validate_xmit_skb_list
validate_xmit_skb
skb_gso_segment
__skb_gso_segment
skb_mac_gso_segment
inet_gso_segment
__udp_gso_segment

	skb_segment
		if (
			hsize <= 0
			&& i >= nfrags
			&& skb_headlen(list_skb)
			&& (skb_headlen(list_skb) == len || sg))

			while (pos < offset + len) {
				  if (i >= nfrags) {
					if (skb_headlen(list_skb))
						BUG_ON(!list_skb->head_frag);

}

== Memory

=== Processes

	arch/x86/kernel/process.c
		sys_execve
			do_execve fs/exec.c
				do_execveat_common
					bprm_execve
						sched_exec
							stop_one_cpu
								cpu_stop_queue_work ->

				open_exec  ▻
				mm_alloc kernel/fork.c
					allocate_mm
						kmem_cache_alloc  ▻
					mm_init
						mm_alloc_pgd
							pgd_alloc arch/i386/mm/pgtable.c. pgd= Page Global Directory
								kmem_cache_alloc  ▻
				SMP sched_exec
				init_new_context
				prepare_binprm
				copy_strings_kernel
				copy_strings
				search_binary_handler
					static struct linux_binfmt *formats;
					load_binary (linux_binfmt)
						load_elf_binary  ▻
					proc_exec_connector

	arch/x86/kernel/process.c
		sys_fork, sys_vfork, sys_clone arch/i386/kernel/process.c
			kernel_clone
			x do_fork kernel/fork.c
				copy_process 400 LOC
					copy_mm
					!CLONE_VM  ▻  dup_mm
					CLONE_VM  ▻
						atomic_inc(&oldmm->mm_users);
					dup_mm	mm_struct
						allocate_mm
						mm_init
						dup_mm_exe_file
							get_mm_exe_file
						dup_mmap
					dup_task_struct task_struct
					copy_thread arch/i386/kernel/process.c
					set_kthread_struct
						p->worker_private = kthread;

		file
		file_operations	ldt
			ioctl	ldt
			O_NONBLOCK	ldt
			splice_read
			mmap	ldt
				generic_file_mmap
					generic_file_vm_ops ??
						filemap_nopage,

		kernel/exit.c
		linux_binfmt
			register_binfmt
			search_binary_handler

			elf_format
				load_elf_library
				load_elf_binary
					open_exec elf_interpreter
						path_lookup_open
							__path_lookup_intent_open
								do_path_lookup
								...
								link_path_walk
									__link_path_walk
										do_lookup
					elf_map
						do_mmap
						do_mmap_pgoff  ▻
					start_thread
				load_elf_interp
				set_brk
					do_brk

			script_format
				load_script
					open_exec

		init_aout_binfmt
			load_aout_binary
				create_aout_tables
				start_thread
		sys_munmap
			do_munmap
		sys_brk	mm/mmap.c
			find_vma
			(do_munmap)
			do_brk_munmap
			do_brk_flags klm
				find_vma_links
				may_expand_vm
				vm_area_alloc
					kmem_cache_alloc mm/slub.c
						__kmem_cache_alloc_lru static
							slab_alloc static
					vma_init
				mas_set_range
				vma_merge
				x find_vma_prepare vm_area_struct
				vma_merge
					get_user_pages
				vma_link
					vma_link_file
					__vma_link
						x __anon_vma_link
					__vma_link_file
						vma_interval_tree_insert via INTERVAL_TREE_DEFINE, near vma_interval_tree_insert_after
							vma_start_pgoff
							vma_last_pgoff

		Chapter 10 Judicious Use of Data Types
		Contents:
			Use of Standard C Types
			Assigning an Explicit Size to Data Items
			Interface-Specific Types
			Other Portability Issues
			Linked Lists

		linux/fs/devfs

=== mmap

	Memory (mmap - ram)

	mm/page-writeback.c
		strictlimit_store
			BDI_CAP_STRICTLIMIT
		wb_position_ratio
			BDI_CAP_STRICTLIMIT

		generic_perform_write <-
			balance_dirty_pages_ratelimited
				balance_dirty_pages
					wb_dirty_limits
						wb_thresh
						wb_bg_thresh

	PFN - Page Frame Number

	include/linux/page-flags.h
		SetPageReserved
			PG_reserved
			set_bit
		ClearPageReserved
			PG_reserved
			__clear_bit

	alloc_pages_exact
		get_order
		__get_free_pages
		make_alloc_exact

	/proc/meminfo
		proc_meminfo_init
			meminfo_proc_fops
			...
				meminfo_proc_show
					si_meminfo  ▻
					si_swapinfo
					get_vmalloc_info
					VMALLOC_TOTAL

	high_memory

	pgd_index	PGDIR_SHIFT	22
	pmd_index	PMD_SHIFT	21
	pte_index	PAGE_SHIFT	12

	early_param	mem
		early_mem	arch/arm/kernel/setup.c
		parse_memopt	arch/x86/kernel/e820.c
			e820_remove_range

	Global Descriptor Table	GDT
	Local Descriptor Table	LDT
	Task State Segment Descriptor	TSSD

	paging_init  ▻

	gdt_page
	get_cpu_gdt_table

	mm_struct->pgd

	pgd_t	PGD	pgd	Page Global Directory	lkm2
	pmd_t	PMD	pmd	Page Middle Directory	lkm2
	pte_t	PTE	pte	Page Table Entry	lkm2	http://en.wikipedia.org/wiki/Page_table
		virt  ▻ phys
		-> page

	tlb	Translation Lookaside Buffer

	arch/x86/mm/tlb.c
	flush_tlb_mm_range
		flush_tlb_multi
			__flush_tlb_multi
				native_flush_tlb_multi
					should_flush_tlb
					flush_tlb_func
					on_each_cpu_cond_mask
			

	86_init	lkm2
		.memory_setup		= default_machine_specific_memory_setup,
			boot_params.screen_info.ext_mem_k

	page	include/linux/mm_types.h

	kmem_cache
		kmem_cache_create 200 LOC
	mempool_s mempool_t
		mempool_create
			mempool_create_node
				kkmalloc_nodemalloc_node
					__kmalloc
	DMA
		/proc/dma

		#include <asm/dma.h>

		davinci_request_dma

			show_mem
			** show_free_areas
				show_free_areas_core
					nr_free_pages

		== pci_alloc_consistent arch/sh/kernel/pci-dma.c
			__get_free_pages gfp_mask=0x21?
				alloc_pages   ▻
					2.4
					_alloc_pages linux/mm/page_alloc.c
						contig_page_data
						__alloc_pages + !!!!!!!!!!!!
							6 __alloc_pages_noprof
							x zone_free_pages pg_data_t # check
								* page = rmqueue(z, order);# alloc
									page = list_entry(curr, struct page, list);
									page = expand(zone, page, index, order, curr_order, area);
								       list_add(&(page)->list, &(area)->free_list);
								        MARK_USED(index, high, area);
									set_page_count
								x _alloc_pages linux/mm/numa.c
									x alloc_pages_pgdat
										pgdat_list
											pg_data_t
									kmalloc
											or
							trace_mm_page_alloc mm_page_alloc
				page_address = (page)->virtual
			dma_cache_wback_inv
				__flush_purge_region
					ocbp

		nr_free_pages
			global_page_state(NR_FREE_PAGES)
				vm_stat lkm2
					NR_VM_ZONE_STAT_ITEMS
					global
					zone->vm_stat
			//for_each_zone pgdat_list free_pages

		== mem
			gup
				get_user_pages
					__gup_longterm_locked
				get_user_pages_fast
					internal_get_user_pages_fast
						gup_pgd_range
							gup_huge_pgd
							gup_huge_pd
						__gup_longterm_unlocked
							__gup_longterm_locked
								__get_user_pages_locked
									__get_user_pages
										get_gate_page
				get_user_pages_fast_only
					FOLL_GET | FOLL_FAST_ONLY
					internal_get_user_pages_fast ->
				pin_user_pages_fast_only
					FOLL_PIN | FOLL_FAST_ONLY
					internal_get_user_pages_fast ->
				unpin_user_pages_dirty_lock
					unpin_user_pages
						unpin_user_page
							__unpin_devmap_managed_user_page
			vmtruncate
			vmtruncate_list
			zap_page_range
				zap_pmd_range
					zap_pte_range
						tlb_remove_page
							__free_pte
								free_page_and_swap_cache
			do_mmap
				mmap_region
			do_munmap
				find_vma_prev
				split_vma
				find_vma
				detach_vmas_to_be_unmapped
				unmap_region
					free_pgtables

				remove_vma_list

			exit_mmap
				free_pgtables

		chr_dev_init
			bdi_init(&zero_bdi);
			devlist
				zero_fops  ▻ /dev/zero
				mem_fops  ▻	/dev/mem
				kmem_fops
					mmap_kmem
			MEM_MAJOR,
			memory_fops
				memory_open
					mem_fops
						write_mem
							do_write_mem
						read_mem
							copy_to_user
						x mmap_mem
							remap_pfn_range
								pmd_alloc
								remap_pmd_range
									remap_pte_range
										pte_alloc_map_lock
											set_pte_at
												native_set_pte_at
													native_set_pte

		== page_mem

		__get_free_pages  ▻

		alloc_page gfp ./include/linux/gfp.h
			alloc_pages gfp_mask order
				numa_node_id()
				alloc_pages_node nid gfp order
					NODE_DATA(nid)		(&contig_page_data)
					__alloc_pages  ▻

				NUMA alloc_pages_current

			__alloc_pages gfp order zonelist
				__alloc_pages_nodemask
					get_page_from_freelist  ▻

				wakeup_kswapd
					zone_watermark_ok
					kswapd_wait
				get_page_from_freelist zonelist lkm2
					extern struct pglist_data contig_page_data
					zonelist
						zone
					buffered_rmqueue
						rmqueue_bulk
							__rmqueue
								__rmqueue_smallest
							__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
						zone_pcp
						bad_range
							page_outside_zone_boundaries
							page_is_consistent
						__mod_page_state
						prep_new_page
						set_page_refs
					zone_statistics

				try_to_free_pages

		== pci_mem
		__pa(x)			- PAGE_OFFSET		(0x80000000UL)
		phys_to_page(__pa(kaddr))
		({ unsigned int node = PHYSADDR_TO_NID(phys); 		      		\
		   NODE_MEM_MAP(node)				 		 	\
		     + (((phys) - NODE_DATA(node)->node_start_paddr) >> PAGE_SHIFT); })

		== pci_free_consistent
				P1SEGADDR 0x8...
				virt_to_page
					phys_to_page(__pa(kaddr))
				free_pages ?
					__free_pages lkm2
						free_hot_page
							free_hot_cold_page
								free_pcppages_bulk
									__free_one_page
								x free_pages_bulk
						__free_pages_ok ***
							page_zone
								zone_table
							list_add(&page->list, &current->local_pages);
							free_one_page
								__free_one_page lkm2
									&zone->free_area[order].free_list lkm2
								__mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
		#include <linux/malloc.h>

		free_area_init
			free_area_init_core
				mem_map	lkm?

		contig_page_data : pglist_data
			zonelist
				zone
					free_area

		x pgdat_list	pglist_data

		kmalloc_array
			__kmalloc
		== kmalloc
			kmalloc_large
				kmalloc_order_trace
					kmalloc_order
						__alloc_pages_nodemask ->
							__alloc_pages_slowpath
							x warn_alloc_failed
			__kmalloc_node
				__do_kmalloc_node	linux/mm/slab.c
				slob_alloc

			kmalloc_sizes.h
		    constexp
			kmalloc_caches
			kmalloc_type
			kmem_cache_alloc_trace
				kmem_cache_alloc lkm2
					slab_alloc ->
		    variable
			__kmalloc - generic
				slub.c
					kmalloc_slab
						kmalloc_caches[kmalloc_type
					slab_alloc
						slab_alloc_node
							__slab_alloc
								___slab_alloc
									get_freelist
							__do_cache_alloc
				slob.c __do_kmalloc_node
				slab.c __do_kmalloc kmem_cache
					__find_general_cachep kmem_cache
						cache_sizes malloc_sizes
					__cache_alloc kmem_cache slab
						--- CONFIG_NUMA alternate_node_alloc
						____cache_alloc array_cache
							cpu_cache_get
							cache_alloc_refill
								array_cache
								cache_grow
									kmem_getpages	slab
										alloc_pages_node  ▻
									alloc_slabmgmt
									set_slab_attr
									init_once
										inode_init_once
								ac->entry

		vm_area_struct

		== kfree lkm2
			kfree source/mm/slab.c
				c = virt_to_cache(objp);
					virt_to_head_page
					page_get_cache
				__cache_free(c, (void *)objp);
			kfree source/mm/slob.c	min	embedded systems
			kfree source/mm/slub.c
				virt_to_head_page
				__free_pages
				slab_free(page->slab, page, object, _RET_IP_);
					do_slab_free
						set_freepointer
						__slab_free

			__kmem_cache_free
				kmem_cache_free_one
					-> cachep->slabs_free

		#include <linux/mm.h>
		GFP_KERNEL
		GFP_ATOMIC
		__GFP_DMA
		__GFP_HIGHMEM

		#include <linux/vmalloc.h>
		void * vmalloc(unsigned long size);

		== vmalloc
				__vmalloc_node mm/vmalloc.c
					__vmalloc_node_range
						__vmalloc_area_node ->

					vm_struct
					get_vm_area_node
						__get_vm_area_node
							vmlist vm_struct
							kmem_cache_alloc  ▻
					__vmalloc_area_node static mm/vmalloc.c
						__vmalloc_node  ▻
						kmalloc_node static
							__kmalloc_node
								__do_kmalloc_node static
									kmalloc_slab
									__kmem_cache_alloc_node
										slab_alloc_node
											__slab_alloc
												___slab_alloc
													new_slab
														allocate_slab

		xxx
					vm_area_alloc_pages
						alloc_pages_node  ▻
						map_vm_area
							vmap_pud_range
								vmap_pmd_range
									vmap_pte_range
										pte_t
										pte_alloc_kernel
											__pte_alloc_kernel
												init_mm
												pte_alloc_one_kernel
		2.4
			__vmalloc
				get_vm_area
					pmd_alloc
						pte_alloc

	x early_trap_init	lkm2
		set_intr_gate(14, &page_fault);
		x page_fault  arch/i386/kernel/entry.S:631: lkm2
			x do_page_fault  current .. mm_structlt	lkm2
				mm_struct mm = tsk->mm;  current .. mm_struct
				?? notify_die notifier_call_chain
				vma = find_vma mm_struct  ▻ vm_area_struct
				handle_mm_fault
					__handle_mm_fault
						handle_pte_fault
							do_swap_page	lkm2
								pte_to_swp_entry
								swap_free
							do_no_page
								filemap_nopage
									find_get_page
										radix_tree_lookup
								mark_page_accessed
							do_wp_page - protection fault
							do_file_page
							do_find_get_page
									alloc_page_vma
										alloc_pages  ▻
									swap_readpage
										get_swap_bio
											map_swap_page
										submit_bio
				mm_fault_error
					out_of_memory
						oom_kill_process
						select_bad_process
						pagefault_out_of_memory
							try_set_system_oom
							out_of_memory
							clear_system_oom

		kswapd
			balance_pgdat
				shrink_zone
					shrink_cache
						shrink_list
							pageout
								swap_writepage
									get_swap_bio
									end_swap_bio_write
										end_page_writeback  ▻

		sys_swapon lkm2
			filp_open
			bd_claim
			read_mapping_page
			setup_swap_extents
				add_swap_extent
						swap_info

		vfree
			__vunmap
				vm_struct
					pages : page
				remove_vm_area
					find_vmap_area
					xx __remove_vm_area
						xx unmap_vm_area
				__free_page linux/gfp.h
					__free_pages  ▻

		<include/asm/uaccess.h>

		copy_to_user()
			_inline_copy_to_user
				raw_copy_to_user
					copy_user_generic
						SMAP - Supervisor Mode Access Prevention
						stac (Set AC flag) / clac (Clear AC flag) 
		copy_from_user
			_copy_from_user
				_inline_copy_from_user
					raw_copy_from_user
						copy_user_generic ->
			__copy_from_user
				__copy_from_user_ll
					__copy_user_zeroing
						__copy_user
							movsl
		get_user
			__get_user_x
				__get_user_
		put_user()

		#include <asm/io.h>
			BUILDIO inb/inw/inl/outb/outw/outl
			kmalloc'ed memory
				virt_to_phys
					__virt_to_phys
						x - PAGE_OFFSET + PHYS_OFFSE
					__pa
						__phys_addr
							v - PAGE_OFFSET
				phys_to_virt
					__va
						+ PAGE_OFFSET
							__PAGE_OFFSET CONFIG_PAGE_OFFSET	0xC0000000
		ioremap  ▻
		iounmap
			arch_iounmap
				__iounmap
					vunmap
						__vunmap  ▻
		ioport_map

		traps.c
			do_address_error
				handle_unaligned_access

		cache  ▻ page_cache

		pci_map_page

		remap_page_range

		flush_cache_all
			CONFIG_SH_CACHE_ASSOC
			CACHE_OC_ADDRESS_ARRAY
			CCR_CACHE_INIT

		CCR		 Address of Cache Control Register

		== sys_cacheflush
			find_vma

		  states
		    coherent, cache = mem
		    contains new data (after write to cache)
		    invalid, necessary (contains no data)
		  operations
		    write back,  cache  ▻ mem,  ▻ coherent
					CACHEFLUSH_D_WB
		      dma_cache_wback       __flush_wback_region  asm ocbwb
		    purge = write back + purge inv, cache  ▻ mem,  ▻ invalid
					CACHEFLUSH_D_PURGE
		      dma_cache_wback_inv   __flush_purge_region  ocbp
		    invalidate
					CACHEFLUSH_D_INVAL
		      dma_cache_inv         __flush_invalidate_region ocbi

		dma_alloc_coherent 2.6
			__get_free_pages  ▻

		davinci_map_io
			davinci_io_desc
		 		{ IO_VIRT,      	IO_PHYS,             IO_SIZE,        	   MT_DEVICE },
						0xe1000000 0x01c00000
				 { DAVINCI_IRAM_VIRT,   DAVINCI_IRAM_BASE,   SZ_16K,               MT_DEVICE },
				 { DAVINCI_FLASH_VIRT,   DAVINCI_ASYNC_EMIF_DATA_CE0_BASE,   SZ_16K,               MT_DEVICE },
				 		0xe2000000	0x02000000
			iotable_init
				create_mapping
					...
					alloc_init_section
					alloc_init_page

		nand_davinci_init
			nand_davinci_probe
				nand_davinci_read_buf
			nandobj.regs = (nandregsovly)IO_ADDRESS (DAVINCI_NAND_BASE_ADDR); 0x01E00000
		    this->IO_ADDR_R   = (void __iomem *) DAVINCI_FLASH_VIRT;

		0x01E00000 0x01E0 0FFF 4K EMIFA Control

		==> ./oo/storage/mem_man/page/notes.txt <==
		struct page l/include/linux/mm_types.h
			pgoff_t index;			/* Our offset within mapping. */
			flags
				ZONES_MASK
				NODES_MASK
				SECTIONS_MASK
				ZONEID_MASK

		linux/mm.h
		remap_page_range
		struct mm_struct *mm = current->mm;
		struct rw_semaphore mmap_sem;
		trvp(page);
		trvx(page_to_phys(page));
		trvx(phys_to_virt(page_to_phys(page)));

		16. Swapping: Methods for Freeing Memory
		      What Is Swapping?
		      Swap Area
		      The Swap Cache
		      Transferring Swap Pages
		      Swapping Out Pages
		      Swapping in Pages
		      Reclaiming Page Frame
	mm/fadvise.c
		sys_fadvise64
			sys_fadvise64_64
	mm/filemap.c
		sys_readahead
	mm/fremap.c
		sys_remap_file_pages
	mm/madvise.c
		sys_madvise
	mm/mempolicy.c
		sys_get_mempolicy
		sys_mbind
		sys_migrate_pages
		sys_set_mempolicy
	mm/migrate.c
		sys_move_pages
	mm/mincore.c
		sys_mincore
			do_mincore
	mm/mlock.c
		sys_mlock2
		sys_mlockall
		sys_mlock
			VM_LOCKED
			do_mlock
				apply_vma_lock_flags
					mlock_fixup
						vma_merge
		sys_munlockall
		sys_munlock
	mm/mmap.c
		sys_brk	->
		sys_munmap
	mm/mprotect.c
		sys_mprotect
		mprotect
			do_mprotect_pkey

			mprotect_fixup
				pgprot_modify

	mm/mremap.c
		sys_mremap
			do_mremap
				mremap_to
				do_munmap
				vma_adjust
				move_vma
	mm/msync.c
		sys_msync lkm2
			vfs_fsync   ▻
	mm/swapfile.c
		swappiness
		sys_swapoff
		sys_swapon  ▻
	show_smap
		walk_page_range
			pgd_offset
				PGDIR_SHIFT
			find_vma
			huge_pte_offset
			walk_pud_range
				pud_offset
				pud_addr_end
				walk_pmd_range
					walk_pte_range
						pte_offset_map
						addr += PAGE_SIZE;
						pte++;
	get_vmalloc_info
		vmlist

}

== Human interface

	include/linux/security.h

		cat /sys/kernel/security/lockdown

		security_locked_down
			call_int_hook
			locked_down

		setup_arch
			security_lock_kernel_down
				call_int_hook
				lock_kernel_down

		lockdown_ops
			lockdown_write
				lock_kernel_down



	v4l2_device_register_subdev
		v4l2_subdev_fops
	security/	include/linux/security.h
		security_init
			security_ops
				register_security

		selinux_init
			x selinux_ops
				selinux_inode_create
					may_create
						permission  ▻
	video4linux
	v4l
		videodev_init
			register_chrdev video_fops
				video_open
					cx8800_devlist
				video_read
					videobuf_read_one
				video_mmap
				video_ioctl
					video_do_ioctl
			class_register(&video_class);
		video_register_device
			video_device
			__video_register_device

		video_ioctl2
			__video_do_ioctl
				VIDIOC_QUERYCAP v4l2_capability
				VIDIOC_QUERYCTRL	v4l2_queryctrl
	sound
	init_soundcore
		soundcore_fops
			soundcore_open
				__look_for_unit
					chains
	snd_soc_skl -  headphones
	snd_hda_intel - internal
	alsa_sound_last_init
		snd_cards
	snd_soc_instantiate_card
	snd_card_register
			snd_device_register_all
				__snd_device_register
		init_info_for_card
		snd_cards
	/proc/asound/cards
	snd_device_register
		__snd_device_register

			esd/mplayer > /dev/dsp c 14 3
			esd ?

		0000:00:02.7 0401: 1039:7012 (rev a0)
		0000:00:02.7 Multimedia audio controller: Silicon Integrated Systems [SiS] Sound Controller (rev a0)
		        Subsystem: C-Media Electronics Inc: Unknown device 0300
		        Flags: bus master, medium devsel, latency 64, IRQ 11
		        I/O ports at dc00 [size=256]
		        I/O ports at d800 [size=128]
		        Capabilities: [48] Power Management version 2

	azx_driver	./sound/pci/hda/hda_intel.c Intel HD Audio
		azx_probe
		snd_card_new
	skl_driver	./sound/soc/intel/skylake/skl.c	ASoC Intel SKL HD Audio driver
		request_threaded_irq

	alsa

		alsa_sound_init
			register_chrdev
			snd_fops
				snd_open

				snd_intel8x0.ko
				intel8x0
				intel8x0_driver ->
					snd_intel8x0_probe ->

		 11:    1192578          XT-PIC  ohci_hcd:usb2, eth0, SiS SI7012

		d800-d87f : 0000:00:02.7
		  d800-d87f : SiS SI7012
		dc00-dcff : 0000:00:02.7
		  dc00-dcff : SiS SI7012

			snd_intel8x0_playback_ops
				snd_intel8x0_playback_open
				snd_intel8x0_pcm_pointer

				x alsa_card_intel8x0_init module_init
					pci_register_driver

					snd_intel8x0_proc_init
						snd_card_proc_new

				        snd_intel8x0_ids
					snd_intel8x0_probe
						snd_card_new
						snd_intel8x0_create
							pci_enable_device PCI
							snd_device_new sound/core/device.c / snd_card_free
							snd_intel8x0_chip_init
								snd_intel8x0_ich_chip_init
							pci_request_regions  ▻
							pci_resource_start
							pci_iomap
							request_irq snd_intel8x0_interrupt -->
							snd_card_set_dev
						snd_intel8x0_mixer
						snd_intel8x0_pcm
						snd_card_register ->
					snd_intel8x0_remove
					intel8x0_suspend
					intel8x0_resume

				request_irq
				snd_intel8x0_interrupt
					snd_intel8x0_update

			pci_driver  ▻ PCI
				snd_audiopci_ids
				snd_audiopci_probe
					snd_card_new kernel.lnk/build/sound/core/init.c

			alsa_card_ens137x_init
				pci_register_driver PCI

			register_sound_dsp
				sound_insert_unit
					__sound_insert_unit
					devfs_mk_cdev SOUND_MAJOR
						devfs_mk_dev
					class_device_create
						class_device_register
						...
							class_device_create_file

	oss
			11:     761694          XT-PIC  ohci_hcd:usb2, eth0, SiS 7012

		d800-d87f : 0000:00:02.7
		  d800-d83f : SiS 7012
		dc00-dcff : 0000:00:02.7
		  dc00-dcff : SiS 7012

					i810_audio.c
						* i810_init_module
							i810_pci_driver
								i810_probe
									register_sound_dsp i810_audio_fops  ▻
					i810_audio_fops
						i810_open
						i810_alloc_pcm_channel
						i810_ioctl
							i810_set_dac_rate
								ac97_set_dac_rate
						i810_write

					+ ac97_codec
						ac97_register_driver
	input
		EV_LED
		hid_process_event
			usb_kbd_event
		atkbd_event
		kbd_init    keyboard kbd

			EV_KEY EV_MSC
			/dev/input/event c 113,  65
			input_register_handler kbd_handler
				input_match_device
				class_device_add class_dev  ▻
				input_table
				handler->connect
				handler->start

		kbd_handler input_handler
			kbd_connect  input_dev
				alloc input_handle
					input_dev
					input_handler
				input_open_device input_handle
					input_dev->open
			kbd_start
				input_inject_event
					input_event ->
			kbd_event
				kbd_keycode
					handle_sysrq  ▻
					k_handler k_self
						put_queue
							tty_insert_flip_char
								tty->flip.char_buf_ptr++ = ch
				tasklet_schedule(&keyboard_tasklet);
					kbd_bh
						input_inject_event
			schedule_console_callback
				console_work, console_callback

		input_init
			register_chrdev  ▻
			input_fops
				input_open_file,
			input_class
				input_dev_release
				input_dev_uevent
					input_add_uevent_modalias_var
						input_print_modalias
			class_register
			input_proc_init

		evdev_init
			evdev_handler
				evdev_fops
					evdev_read
						input_event ->
					evdev_write
				evdev_connect
					input_register_handle
					evdev_install_chrdev

		mousedev_init
			INPUT_MAJOR     13
			hexdump /dev/input/mice
			# 13,  63
			mousedev_handler
			input_register_handler input_handler  ▻
			class_device_create  ▻

			misc_register
				device_create  ▻
			misc_deregister
				device_destroy

		mousedev_handler input_handler
			mousedev_event  ▻
			mousedev_connect input_dev
				input_open_device  ▻
				class_device_create  ▻
			mousedev_fops
				mousedev_read sys_read mousedev_read
					mousedev_packet
		usbhid_probe
			static
			usb_register
			hid_driver
			hid_add_device

		module_hid_driver

		hidinput_getkeycode
			hidinput_locate_usage

		input_default_getkeycode
			input_fetch_keycode
		keycode
		keycodesize
		keycodemax;

		input_configured

		hid_input_report
			hid_report_raw_event
				...
					hid_usage
					hid_process_event
					KEY_RESERVED KEY_A ... KEY_MAX

		hid_device

		common_interrupt
			__irq_exit_rcu
				local_softirq_pending
				invoke_softirq
				__do_softirq
					tasklet_action_common
						usb_giveback_urb_bh
							__usb_hcd_giveback_urb➝hid_irq_in➝hid_input_report➝hid_report_raw_event
								hid_input_array_field
									hid_process_event
										event
										hidinput_hid_event
										hiddev_hid_event
											hiddev_send_event

		hid_driver semitek_driver
			my semitek_event
				input_report_key

		lg_ultrax_remote_mapping ts_input_mapping

		hid_debug_init hid_debug_register

		KEY_HOME
		KEY_END

		usb_kbd_init NOT  USED
			usb_kbd_driver
				... USB_INTERFACE_PROTOCOL_KEYBOARD
				usb_kbd_probe
					usb_kbd
						input_dev
					input_dev = input_allocate_device
					input_register_device(kbd->dev); input_dev
						class_device_add  ▻
						input_handle handler->connect
						input_dev_list
						input_handler_list
						input_dev = input_allocate_device

						    input_dev->event = usb_kbd_event
								usb_submit_urb
							input_dev->open = usb_kbd_open
								usb_submit_urb
					usb_set_intfdata
			usb_kbd_irq
				input_report_key

		usb_mouse_init NOT USED
			usb_mouse_driver
				usb_mouse_probe
					input_dev = input_allocate_device
		tty_class_init
			tty_class = class_create
		tty_register_device
			tty_class
			device_create
		tty_init
			device_create(tty_class, NULL, MKDEV(TTYAUX_MAJOR, 1), "console");
			TTYAUX_MAJOR
			cdev_add
				tty_fops
					tty_read  ▻
					tty_write <-vfs_write <-
						do_tty_write  drivers/char/tty_io.c
							copy_from_user
							tty_ldisc->write
								write_chan	drivers/char/n_tty.c
									opost_block
										con_flush_chars
										con_write
											do_con_write
												scr_writew out to screen
									con_write_room
									con_flush_chars
										set_cursor drivers/char/vt.c
											add_softcursor
											vgacon_cursor vgacon.c
												write_vga
												vgacon_set_cursor_size
											con_start
									release_console_sem
										call_console_drivers

			register_chrdev_region /dev/tty
			device_create TTYAUX_MAJOR

			cdev_init(&console_cdev, &console_fops);
				console_fops
					tty_open
						console_device
							console_drivers
					tty_read drivers/char/tty_io.c:
						tty_ldisc_ref_wait "Line Discipline"
						tty_ldisc_try
						ld->read
							read_chan drivers/char/n_tty.c
								job_control
								read_buf

			cdev_add console_cdev
			    register_chrdev_region /dev/console

		tty?
			do_sys_open  ?
				filp_open
					file_open_name
						do_filp_open  ▻
					x nameidata_to_filp
						__dentry_open
							chrdev_open
								tty_open
									init_dev
										n_tty_open

		register_console

		flush_to_ldisc drivers/char/tty_io.c disc->receive_buf
			tty->flip.char_buf  ▻
			-> n_tty_receive_buf drivers/char/n_tty.c
				? n_tty_receive_char
					put_tty_queue_nolock
						put_tty_queue
							read_buf <= c

		kgdb8250_add_platform_port

		tty_struct
			read_buf

		i8042_init
			i8042_platform_init
			platform_driver_register i8042_driver
			i8042_create_aux_port
				i8042_port_register
			i8042_create_kbd_port
				i8042_open ( on insert)
					request_irq
					i8042_interrupt
						i8042_read_data
							return inb(I8042_DATA_REG);
						serio_interrupt
							atkbd_interrupt
								input_event
								atkbd_report_key
									input_report_key
										input_event
											atkbd_event ./drivers/input/keyboard/atkbd.c
												schedule_work atkbd_event_work
													ps2_command ??
											kbd_event  ▻
								psmouse_interrupt
									psmouse_process_byte
										input_report_rel
											input_event
												mousedev_event mousedev.c:296
													mousedev_rel_event
														mousedev->packet.dx
													mousedev_notify_readers
										input_report_key ->

			i8042_timer_func

		./arch/i386/kernel/head.S

		sysenter_entry
			sysenter_past_esp

		proc_kmsg_operations
			kmsg_read vfs_read
				do_syslog

		/dev/snd
		/dev/dsp
			snd_pcm_oss_write sound/core/oss/pcm_oss.c  <- vfs_write sys_write
				snd_pcm_oss_write1
					snd_pcm_oss_make_ready
						snd_pcm_oss_change_params
							snd_pcm_kernel_ioctl
							....
								snd_intel8x0_setup_pcm_out

			snd_intel8x0_interrupt

		/dev/video

		cam

		sys_ioctl		fs/ioctl.c lkm2
			do_vfs_ioctl
				file_ioctl
				vfs_ioctl
					filp->f_op->unlocked_ioctl
						ext4_ioctl
						v4l2_unlocked_ioctl

					lock_kernel
					filp->f_op->ioctl
						autofs_root_ioctl
						i2c_ioctl
						smb_ioctl
						fat_generic_ioctl
						ata_scsi_ioctl
						tcp_ioctl
						inet_ioctl
						video_ioctl2
					unlock_kernel

		sys_select
			core_sys_select
				do_select
					poll
						pipe_poll
							__pollwait fs/select.c:101
								__get_free_pages -- frequent
					poll_freewait
						free_pages

		cache_reap
			drain_array_locked
				free_block
					slab_destroy
						kmem_freepages	slab
							free_pages
							virt_to_page
							__free_pages  ▻

		== procfs
			proc_file_read <- vfs_read

		== fb
			fbmem_init
				register_chrdev
				fb_ops

						register_framebuffer
							device_create  ▻
							registered_fb
							fb_init_device
								device_create_file  ▻
						fbmem_read_proc
						fb_write
							fb_get_fix
		fbcon_setup

		consw fb_con
			fbcon_init

		fb_console_init
			fbcon_takeover
				take_over_console

		fbcon_putcs
		get_color
			vc_data

		display fb_display

		sisfb_init_module
			sisfb_init
				sisfb_setup
				sisfb_driver
					sisfb_probe
						ivideo
							sisfb_inverse
						sisfb_ops
						sisfb_set_disp
						display
		vt8623fb_init	linux-2.6.22.9/drivers/video/vt8623fb.c
			fb_get_options("vt8623fb", &option))
			pci_register_driver	vt8623fb_pci_driver
				vt8623_pci_probe
					pci_request_regions  ▻
					framebuffer_alloc + vt8623fb_info
					register_framebuffer
					vt8623fb_ops fb_ops
						vt8623fb_open
						vt8623fb_imageblit
				pci_iomap
		== console
		vc vc_cons
			vc_data
		con_init
			save_screen
			update_screen

		console_init

	drivers/gpu
		DRM_IOCTL_BASE
		drm_init
			drm_driver	lkm2
			drm_platform_init
				drm_get_platform_dev
			drm_pci_init
				pci_register_driver

		radeon_init
			drm_init

		i915_init
			drm_init
		nouveau_init
			drm_init

		amdgpu_init
		hl_init habanalabs_drv.c
			hl_pci_driver
				hl_pci_probe
				hl_pci_remove
				hl_device_init
					device_early_init
						goya_set_asic_funcs
							goya_funcs
								goya_early_init
				hl_pm_ops
					hl_pmops_suspend
						hl_device_suspend
							pci_save_state
							pci_disable_device
							pci_set_power_state
				hl_pci_err_handler
					hl_pci_err_detected

	drivers/media/video/
		uvc_init
			uvc_driver
				uvc_probe
					uvc_register_video
						video_device
						video_device_alloc
						video_register_device  ▻
			gspca_init
			gspca_dev_probe
				gspca_dev_probe2
					video_register_device  ▻

			sd_mod_init
				usb_register(&sd_driver);
				sd_driver	(many)
					sd_probe  ▻
						gspca_dev_probe  ▻

	serial8250_isa_driver
		serial8250_probe
			plat_serial8250_port
			serial8250_register_port
				uart_add_one_port
					tty_register_device

=== video

	uvc_mc_init_entity
		media_entity_init

	g_webcam from 2.6.35
	uvc_gadget_trace_param	drivers/usb/gadget/f_uvc.c
		UVC_TRACE_PROBE
		uvc_trace

	uvc_trace_param

	webcam_init	drivers/usb/gadget/webcam.c
		return usb_composite_probe(&webcam_driver, webcam_bind);
			webcam_bind
				usb_add_config(cdev, &webcam_config_driver, webcam_config_bind)
					webcam_config_bind
						uvc_bind_config	drivers/usb/gadget/f_uvc.c
							uvc_device	drivers/usb/gadget/uvc.h
							usb_add_function
								bind
									uvc_function_bind
										uvc_video_init	--- drivers/media/video/uvc/uvc_video.c
												drivers/usb/gadget/uvc_video.c
										uvc_register_video drivers/usb/gadget/f_uvc.c
											video_device_alloc
											uvc_v4l2_fops
												uvc_v4l2_open->
												uvc_v4l2_ioctl  ▻

												uvc_v4l2_mmap
											video_register_device
		usb_add_gadget_udc  ▻
	usb_composite_probe
		usb_gadget_probe_driver
			&udc_list
		composite_bind

	uvc_v4l2_open
		uvc_function_connect
			usb_function_activate
				usb_gadget_connect
					gadget->ops->pullup  ▻  ? musb_gadget_pullup

	uvc_v4l2_ioctl
		uvc_v4l2_do_ioctl
			uvc_v4l2_set_format

	camorama -D --width=640 --height=480

	stk_camera_init	./drivers/media/video/stk-webcam.c
		usb_register  ▻
		stk_camera_driver
			stkwebcam_table
			stk_camera_probe
				stk_register_video_device
					stk_v4l_data
						v4l_stk_ioctl_ops
							stk_vidioc_s_fmt_vid_cap
								stk_vidioc_try_fmt_vid_cap
								stk_initialise ~= dev_stk11xx_initialize_device
									stk1125_initvals	~= dev_stk0408_configure_device
									stk_camera_write_reg
										usb_control_msg
								stk_setup_format
									stk_sizes
									stk_sensor_configure
							stk_vidioc_streamon
								stk_start_stream ~= dev_stk0408_start_stream
									stk_camera_read_reg
									stk_camera_write_reg
						v4l_stk_fops
							v4l_stk_read
								stk_initialise ~= dev_stk11xx_initialize_device
							v4l_stk_open
							video_ioctl2
								__video_do_ioctl
					video_register_device  ▻

	syntekdriver
		2.6.31-16-generic OK
			rmmod stk11xx; modprobe stk11xx norm=1
			0=NTSC, 1=PAL
		usb_stk11xx_init
			usb_register
			usb_stk11xx_driver
				usb_stk11xx_probe
					dev_stk11xx_camera_on
					:0501 SYNTEK_STK_M811
					05e1:0408 Syntek Semiconductor Co., Ltd
					udev->descriptor.idVendor udev->descriptor.idProduct
					webcam_model
					webcam_type

					dev_stk11xx_initialize_device
						dev_stka311_initialize_device
						dev_stk0408_initialize_device
							usb_stk11xx_write_registry
							dev_stk0408_configure_device
								dev->vsettings.norm
								dev_stk0408_set_resolution
					v4l_stk11xx_register_video_device  ▻
					usb_stk11xx_default_settings
						dev->vsettings.norm

					v4l_stk11xx_register_video_device
						video_register_device  ▻
						v4l_stk11xx_fops

			v4l_stk11xx_fops
				v4l_stk11xx_open
					v4l_stk11xx_select_video_mode
					dev_stk11xx_init_camera
						dev_stk0408_init_camera
							dev_stk0408_camera_asleep
							dev_stk0408_configure_device
								usb_stk11xx_read_registry
								usb_stk11xx_write_registry
								dev_stk0408_sensor_settings
					dev_stk11xx_camera_on
						usb_set_interface
					dev_stk11xx_reconf_camera
					err = usb_stk11xx_isoc_init
					dev_stk11xx_start_stream  ▻
					dev_stk11xx_camera_settings
						dev_stk0408_camera_settings
							dev_stk0408_set_camera_quality
				v4l_stk11xx_read
					stk11xx_handle_frame
				v4l_stk11xx_ioctl
					VIDIOC_DQBUF
						stk11xx_handle_frame
							dev_stk11xx_decompress
								dev_stk0408_decode
								stk11xx_copy_uvyv
								stk11xx_copy_rgb
					video_usercopy
					v4l_stk11xx_do_ioctl
						VIDIOC_STREAMON	usb_stk11xx_isoc_init
						VIDIOC_STREAMOFF	usb_stk11xx_isoc_cleanup
						stk11xx_next_image
						dev_stk11xx_init_camera
						dev_stk11xx_camera_on
						dev_stk11xx_reconf_camera
						dev_stk11xx_start_stream
							dev_stka311_start_stream
							dev_stk0408_start_stream
						dev_stk11xx_camera_settings
			video_set_drvdata
						video_register_device  ▻

	emac_dev_xmit
		emac_send
			emac_write

	usbtest_init
		usbtest_driver
			usbtest_ioctl

	usb_serial_generic_register
		usb_serial_register	usb_serial_generic_device
		usb_register		generic_driver

==== gpu

	ast_init
		ast_pci_driver
			ast_pci_probe
				ast_driver
	ast_pci_driver

===== mgag200

	mgag200_pci_driver
		mgag200_driver

	.vmap      = drm_gem_vram_object_vmap
		drm_gem_vram_vmap
	drm_gem_shmem_funcs
		.vmap = drm_gem_shmem_object_vmap
	drm_clflush_pages
		drm_cache_flush_clflush
			drm_clflush_page
				clflushopt
	mgag200_handle_damage
		drm_gem_shmem_vmap
		drm_fb_memcpy_dstclip
	 	6 drm_fb_memcpy
		+ drm_clflush_virt_range
			clflushopt
				clflush	Flush Cache Line
			wbinvd_on_all_cpus
				on_each_cpu
				wbinvd
					native_wbinvd
						asm wbinvd
		mgag200_set_startadd
		mgag200_set_offset

	gem_create_object = mgag200_create_object

===== drm

	drm_gem_object
		drm_gem_object_funcs
	: {{The Linux Kernel/man|2|membarrier}}
	arch/x86/include/asm/cacheflush.h
		clflush_cache_range arch/x86/mm/pat/set_memory.c
			mb arch/x86/include/asm/barrier.h
				asm "mfence" : : : "memory"
			clflush_cache_range_opt
				CPA - Change Page Attribute
				clflushopt
			mb

	x drm_gem_fb_simple_display_pipe_prepare_fb
		x drm_gem_fb_prepare_fb

	drm_simple_display_pipe_funcs
		DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS
			.prepare_fb
			drm_gem_simple_kms_prepare_shadow_fb
				drm_gem_vmap
					vmap

	drm_gem_shmem_vmap
		drm_gem_shmem_vmas_locked
			map_wc
			pgprot_writecombine ->

	drm_gem_shmem_object
		map_wc
		pgprot_writecombine
			__pgprot(pgprot_val(prot) | cachemode2protval(_PAGE_CACHE_MODE_WC));


=== Debug


include/linux/ftrace.h

	cd /sys/kernel/tracing/
	set_event				# set event filter (*:* means all events)
	tracing_cpumask		# cpus 2-127 (exclude housekeeping 0-1)
	current_tracer	# choose the function_graph tracer
	max_graph_depth				# how many levels are shown (0 means all)
	options/latency-format		# show latency information
	options/funcgraph-abstime		# display timestamp from boot at each line
	options/funcgraph-tail		# show function end instead of just "}" 
	options/funcgraph-proc		# show process' command line at each line
	tracing_on					# start writing to the trace ring buffer



kernel/trace/trace_irqsoff.c

	CONFIG_PREEMPT_TRACER=y
		trace_preempt_off
				tracer_preempt_off
					start_critical_timing
		trace_preempt_on
			tracer_preempt_on
				preempt_trace
				stop_critical_timing
					__trace_function ->
					check_critical_timing ->

	include/linux/tracepoint.h
	include/trace/define_trace.h
	include/trace/trace_events.h
		TRACE_EVENT
			DECLARE_TRACE*

include/linux/ftrace.h
kernel/trace/trace_preemptirq.c

	CONFIG_TRACE_PREEMPT_TOGGLE
	trace_preempt_on
	trace_preempt_off

include/trace/events/preemptirq.h

	#define TRACE_SYSTEM preemptirq
	CONFIG_TRACE_IRQFLAGS
	trace_irq_enable
	trace_irq_disable
	CONFIG_TRACE_PREEMPT_TOGGLE
	trace_preempt_enable
	trace_preempt_disable

kernel/trace/trace_irqsoff.c

	init_irqsoff_tracer
		irqsoff_tracer
			irqsoff_tracer_init
				__irqsoff_tracer_init
					set_tracer_flag
					ftrace_init_array_ops
					start_irqsoff_tracer
						register_irqsoff_function
							register_ftrace_graph
							register_ftrace_function
								prepare_direct_functions_for_ipmodify
								register_ftrace_function_nolock
		preemptoff_tracer
			preemptoff_tracer_init
				__irqsoff_tracer_init ->
		preemptirqsoff_tracer
			preemptirqsoff_tracer_init
				__irqsoff_tracer_init ->
			irqsoff_tracer_start
	./tools/tracing/latency/latency-collector.c
	CONFIG_IRQSOFF_TRACER=y
		tracing_max_lat_fops
			tracing_max_lat_write
				max_latency
			d_max_latency
				echo 0 > /sys/kernel/tracing/tracing_max_latency
		trace_hardirqs_off_finish
			tracer_hardirqs_off
				start_critical_timings
					start_critical_timing
						ftrace_now
							buffer_ftrace_now
								ring_buffer_time_stamp
									rb_time_stamp
										trace_clock_local ->
						__trace_function
							trace_graph_function
								__trace_graph_function
									__trace_graph_entry
										__graph_entry
											ring_buffer_event_data
									__trace_graph_return
										ring_buffer_event_data
							trace_function
								ring_buffer_event_data
									rb_event_data
		trace_hardirqs_on
			tracer_hardirqs_on
				stop_critical_timings
					stop_critical_timing
						__trace_function ->
						__trace_stack ->
						check_critical_timing
							ftrace_now
							report_latency
								tracing_thresh
								max_latency
							__trace_function ->
	default_idle_call, cpu_idle_poll
		start_critical_timings
	DEFINE_EVENT
	DECLARE_TRACE*
	__DECLARE_TRACE*
	__DECLARE_TRACE_COMMON
		register_trace_##name

	vi /etc/default/grub; update-grub; reboot

	# argv = "/usr/bin/gdb -q -batch -ex run -ex bt --args".split() + argv

	do_int3
		do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
		-->int3 @ arch/x86/kernel/entry_32.S
		?? do_int3

	pr_debug
		#undef pr_fmt
		#define pr_fmt(fmt)    "%s.c:%d %s " fmt, KBUILD_MODNAME, __LINE__, __func__

			pr_debug("\n");
			pr_debug("ret = %d\n",ret);
			pr_info
			on linux-5 see mtrr_rendezvous_handler
			pr_devel

	ctracer
		KBUILD_CFLAGS:= $(filter-out -Werror-implicit-function-declaration,$(KBUILD_CFLAGS))
		KBUILD_CFLAGS:= $(filter-out -Wdeclaration-after-statement,$(KBUILD_CFLAGS))
		ccflags-y+= -D CTRACER_ON -include linux/printk.h -include linux/kern_levels.h -include ctracer.h
		subdir-ccflags-y+= -D CTRACER_ON -include ctracer.h
		ccflags-y+= -DDEBUG
		subdir-ccflags-y = -D DEBUG

		dynamic_debug_init_debugfs

		dev_dbg
				#undef CONFIG_DYNAMIC_DEBUG
				#define DEBUG
				dynamic_dev_dbg
				dev_printk
		kdb
			kdb_init(KDB_INIT_FULL);
				kdb_inittab
				kdb_initbptab
				kdb_cmd_init
					kdb_cmds

			kdb_stub

			echo ttyS0 > /sys/module/kgdboc/kernel/kgdboc
			echo kbd > /sys/module/kgdboc/kernel/kgdboc
			echo g > /proc/sysrq-trigger

			kdb_inittab

		kprobes
			init_kprobes
				arch_init_kprobes

		register_kretprobes
			register_kretprobe
				register_kprobe lkm2
					arch_prepare_kprobe
						arch_copy_kprobe

		URL=http://lkcd.sourceforge.net/

		==> ./stages/testing/debug/crash/notes.txt <==
		do_page_fault
		die
			show_registers
			show_regs
				show_stack
					stack = esp
					show_trace
						lookup_symbol
			do_exit  ▻
		panic
			dump_stack
				dump_stack_lvl
					show_stack
						trace_nvme_complete_rq
						show_trace_log_lvl
							printk_stack_address
				x dump_backtrace
					x unwind_backtrace
						unwind_frame
						dump_backtrace_entry
							x print_symbol

==== oops

		kgdb  ▻
		do_page_fault
		show_state

		debugging
		CONFIG_MAGIC_SYSRQ
			proc_sysrq_trigger_operations
			handle_sysrq
				sysrq_key_table
					GDB_OP
					sysrq_handle_showstate
						show_state
							show_state_filter
								sched_show_task
							x show_task
						show_workqueue_state ->

			2.4
			handle_sysrq
				__handle_sysrq_nolock
					__sysrq_get_key_op
						sysrq_key_table
		/* 1 */	&sysrq_loglevel_op,
		/* b */	&sysrq_reboot_op, sysrq_handle_reboot
		/* e */	&sysrq_term_op, sysrq_handle_term
		/* g */	GDB_OP, breakpoint BREAKPOINT
		/* h */	NULL,
		/* i */	&sysrq_kill_op, sysrq_handle_kill
		/* k */	&sysrq_SAK_op,
		/* m */	&sysrq_showmem_op,   show_mem
		/* p */	&sysrq_showregs_op, sysrq_handle_showregs show_regs
		/* r */	&sysrq_unraw_op,
		/* s */	&sysrq_sync_op,
		/* t */	&sysrq_showstate_op, sysrq_handle_showstate show_state
		/

==== kgdb

		traps.c
		CHK_REMOTE_DEBUG
			kgdb_debug_hook  ▻ kgdb

			arm-none-linux-gnueabi-gdb vmlinux

			target remote /dev/ttyUSB0

		http://kgdb.sourceforge.net/
		http://kgdb.linsyssoft.com/

		x kgdb_initialized
			kgdb_breakpoint
			kgdb_early_entry
				kgdb_arch_init
					kgdb_internal_init
				kgdb_io_ops.init
					init_kgdboe kgdb_eth.c
					kgdb8250_init
						serial_outb

		kgdb8250_local_init
			serial_outb = io_outb

		CONFIG_KGDB
		CONFIG_KDB
		x kgdb_init
		breakpoint
		do_page_fault
		__do_page_fault
			oops  ▻
			kgdb_bus_err_hook
			force_sig_info_fault

		kgdb_parse_options
			kgdb_debug_hook = kgdb_handle_exception;
		 breakpoint
		 BREAKPOINT

		CONFIG_FRAME_POINTER

		kgdb_io kgdb_io_ops

		==> ./stages/testing/debug/knotes.txt <==
		-> kgdb

		== log
			control_devkmsg
			___ratelimit
			memdev
				kmsg_fops
					devkmsg_release
						ratelimit_state_exit
					devkmsg_write
					- kmsg_write

==== printk
			print_hex_dump
			pr_crit_ratelimited
			dprintk --- various implementations

			default_message_loglevel DEFAULT_MESSAGE_LOGLEVEL
			console_printk
			printk_ratelimit
				__printk_ratelimit
			log_buf
				log_buf_len_setup
				__log_buf __LOG_BUF_LEN CONFIG_LOG_BUF_SHIFT
				emit_log_char
					LOG_BUF
						log_buf
			do_syslog
			call_console_drivers
				_call_console_drivers console_loglevel DEFAULT_CONSOLE_LOGLEVEL
					__call_console_drivers
						console_drivers
							con->write
			printk_once
			vprintk_default
				vprintk_emit
					vscnprintf

	trace_printk
		do_trace_printk
			__trace_bprintk
				trace_vbprintk
					vbin_printf
			__trace_printk
				trace_vprintk
					__trace_array_vprintk
						vscnprintf ->
		trace_puts
	printk_deferred
		printk_index_wrap
			__printk_index_emit
			_printk_deferred
				vprintk_deferred
					vprintk_emit
						vprintk_store
						printk_store perf
							vsnprintf


	
	irq_work/7-91      [007] .....  3489.531424: noise: irq_work_single wake_up_klogd_work_func duration 50 us

	vprintk_emit
		wake_up_klogd
			__wake_up_klogd
				wake_up_klogd_work
					wake_up_klogd_work_func
						<- irq_work_single
						printk_pending
						wake_up_interruptible(&log_wait);
							__wake_up_common


	printk
		/proc/sys/kernel/printk
			debug_kernel
			quiet_kernel
		vprintk
			CONFIG_DEBUG_LL
			printascii
				senduart
			vscnprintf
				vsnprintf
					format_decode
						bitmap_string
					kstrtoint
					simple_strtol
					simple_strtoul
					number

		klogd

		KERN_EMERG 0  .. KERN_DEBUG 7

		sys_syslog
			do_syslog

		seq_file
			seq_operations

		console_verbose
			console_loglevel
				console_printk

		log_start

		CONFIG_PROFILING

		proc  - discouraged

		URL=http://ltp.sourceforge.net/ https://github.com/linux-test-project/ltp
			ltp
				ltpmenu
				runltp
					scenario_groups/default
				runltplite.sh
					./runtest/ltplite
		CREATE_ENTRIES=1 time ./runltplite.sh -q -d /var/volumes/vol1/tmp | tee $(uname -r).ltplite.txt
		cd /var/volumes/vol1/ltp
		CREATE_ENTRIES=1 time ./runltp -q -d /var/volumes/vol1/tmp | tee $(uname -r).ltp.txt

		https://codeload.github.com/pjd/pjdfstest/zip/master

		/fs/devfs/base.c depricated
			devfs_mk_cdev
				devfs_mk_dev


==== oprofile

		oprofile_operations
		oprofile_init lkm2
			oprofile_arch_init
				op_nmi_init
					op_amd_spec
					op_ppro_spec
					nmi_start
						nmi_cpu_start
							op_amd_start
							ppro_start
								rdmsr
								rdmsrl
									native_read_msr
										__rdmsr
										do_trace_read_msr
								wrmsrl
									wrmsr
				nmi_timer_init
					timer_start
						nmi_timer_callback
							oprofile_add_sample
				x86_backtrace
					dump_backtrace
						oprofile_add_trace
							add_sample
			oprofilefs_register
				oprofilefs_type
					oprofilefs_get_sb
						oprofilefs_fill_super
							oprofile_create_files
								enable_fops
									enable_write
										oprofile_start lkm2
											nmi_start  ▻
	quiet
	+ cat /proc/sys/kernel/printk
	10      4       1       7

==== bpf

	__sys_bpf - syscall entry
		...
		bpf_prog_load - load program
  			bpf_check - verifier

	kern_sys_bpf
		bpf_prog_run - execute program

include/linux/filter.h

	bpf_jit_enable

	bpf_int_jit_compile

		bpf_jit_binary_lock_ro
			set_memory_ro
			set_memory_x
			set_memory_rox
				change_page_attr_set_clr


	bpf_jit_harden W^X
		sysctl net.core.bpf_jit_harden


  # echo 'hist:keys=delta.buckets=100,stack.stacktrace:sort=delta' > events/synthetic/block_lat/trigger
  # cat events/synthetic/block_lat/hist

  #
	./bcc/tools/xfsslower.py
		bpf_ktime_get_ns
		xfs_file_fsync
		min_ms
		FILTER_US

	./bcc/examples/tracing/biolatpcts.py
		io_start_time_ns
		bpf_ktime_get_ns

	bcc/tools/biolatpcts.py
		io_start_time_ns

	./bcc/examples/tracing/disksnoop.py
		bpf_ktime_get_ns
		bpf_trace_printk

	./bcc/libbpf-tools/biolatency.c
		bpf_program__set_autoload
		progs.block_rq_complete

	./bcc/libbpf-tools/biolatency.bpf.c
		trace_block_rq_insert
		block_rq_issue
		trace_rq_start
			bpf_map_update_elem(&start, &rq, &ts, 0);
		block_rq_complete
			handle_block_rq_complete
				bpf_map_update_elem
				bpf_map_lookup_elem

	./bcc/libbpf-tools/biosnoop.c
		min_lat_ms
		perf_buffer__new
		bpf_program__set_autoload
		bpf_program__set_attach_target

	./bcc/libbpf-tools/biosnoop.bpf.c
		block_rq_insert
			trace_rq_start
				bpf_map_update_elem(&start, &rq, stagep, 0);
		block_rq_complete
			min_ns
			bpf_perf_event_output

	./bcc/tools/biolatency.py - BPF_HISTOGRAM

	./bcc/libbpf-tools/wakeuptime.bpf.c
	./bcc/tools/wakeuptime.py
	./bcc/tools/runqslower.py

==== bpf internals

	samples
		bpf_helpers.h __BPF_HELPERS_H
		tracex4_kern
			bpf_prog2
				PT_REGS_RC
		tracex2_kern
			bpf_prog3 kprobe/sys_write
		tracex5_kern
		bpf_prog1
			bpf_probe_read
			syscall_nrs
			syscall_defines
				SYS__NR_write
					bpf_trace_printk
		syscall_tp_user
			load_bpf_file
				do_load_bpf_file
					get_sec
						elf_getdata
					kprobe/
					tracepoint/
			bpf_map_lookup_elem
		syscall_tp_kern
			trace_enter_open

	make O=/home/costa/App/Platforms/X86/ bpf  -C tools/
	tools/bpf/Makefile
	tools/bpf/bpftool/Makefile
		tools/lib/bpf/Makefile
	user
		lib
		bpf_raw_tracepoint_open BPF_RAW_TRACEPOINT_OPEN
			bpf_prog_get_type
				bpf_prog_get_type_dev
					__bpf_prog_get
						____bpf_prog_get
					trace_bpf_prog_get_type
			bpf_probe_register
				__bpf_probe_register
					tracepoint_probe_register
						tracepoint_probe_register_prio
							tracepoint_add_func
								func_add
		do_batch
		do_prog
		do_load
			bpf_prog_load
				BPF_PROG_LOAD
				bpf_object__load
					bpf_object__load_progs
						bpf_program__load
							load_program
								bpf_load_program_name
									sys_bpf
									BPF_PROG_LOAD
	sys_bpf
		BPF_PROG_LOAD
		bpf_prog_load
			find_prog_type
				BPF_PROG_TYPE_TRACEPOINT
				BPF_PROG_TYPE_RAW_TRACEPOINT
				BPF_PROG_TYPE_KPROBE
				BPF_PROG_TYPE_PERF_EVENT
			bpf_obj_name_cpy
			bpf_prog_select_runtime
			bpf_prog_kallsyms_add
	bpf_attr
	pf_trace_printk
	bpf_get_current_comm
	bpf_perf_prog_read_value
	bpf_trace_printk_proto
		bpf_trace_printk BPF_CALL_5
	SEC("kprobe/sys_open")
	bpf_sys_open
		bpf_probe_read_str
	bcc
		BPF_PERF_OUTPUT
		events.perf_submit
		bpf_ctx["BPF_PERF_OUTPUT"].open_perf_buffer(print_event)
		perf_buffer_poll
	detach_kprobe_event
		bpf_detach_kprobe
			bpf_detach_probe
        TestKprobe
		HelloWorld
			BPF.init
				load_string
					load_cfile
			BPF_PROGRAM
			attach_kprobe
				load_func
					bpf_prog_load  ▻
				bpf_attach_kprobe
					bpf_attach_tracing_event
						__NR_perf_event_open
			get_syscall_fnname
			detach_kprobe
		get_syscall_fnname
		attach_kprobe
		bpf_attach_kprobe

		trace_print src/python/bcc/__init__.py
			trace_readline
				trace_open
					trace_pipe
		kprobe__sys_clone
		bpf_usdt_readarg

== other
{

	other
		audit_init
			netlink_kernel_create(&init_net, NETLINK_AUDIT, 0, audit_receive, NULL, THIS_MODULE);
				audit_receive
					audit_receive_skb
						audit_receive_msg
							kauditd_thread

		== lists
			struct list_head
				list_add
				LIST_HEAD
				list_for_each_entry
				list_for_each_entry_reverse ??
			list_head
				list_add_tail
		== config

		qconf
			configApp

			conf_parse
			conf_read
			ConfigMainWindow

		gtk
			glib
			pango

		==> ./deb <==
		fakeroot make-kpkg --initrd --revision=2.6.15.1-conan kernel_image
		make-kpkg --initrd --revision=2.6.15.1-conan kernel_image
		dpkg -i linux-image-2.6.15.1_2.6.15.1.conan_i386.deb

		==> ./snd-remove <==
		rmmod snd snd_ens1371 snd_rawmidi snd_seq_device snd_ac97_codec snd_pcm_oss snd_mixer_oss snd_pcm snd_timer

		==> ./linked_list_pp <==
		grep -Prn ' *= *&\w+ *-> *next *;' kernel.lnk/linux-2.6.13-conan/

		==> ./grep_dm <==
		grep ^exec /var/log/dmesg | sort | uniq -c | sort -n

http://en.wikipedia.org/wiki/Linux_kernel[Linux kernel]

}

=== git

git branch --edit-description

git format-patch -3 --cover-letter --cover-from-description=subject

git fetch origin merge-requests/$mrid/head:$branchname

git describe --contains  --match 'v*' 5a80bd075f3b

=== common

	ICA	InterCallings Analysis	http://tldp.org/HOWTO/KernelAnalysis-HOWTO-2.html
	ctags -x -R include/ | while read a b c; do echo $b; done | sort | uniq -c
	ctags -x --c-kinds=-mde+p -R include/ | while read -a a ; do echo -e "${a[3]} ${a[1]} \t${a[0]}"; done
	ctags -x --c-kinds=d -R include/ | while read -a a ; do echo -e "${a[0]}"; done > include-defines.ids

	grep pci include-defines.ids | while read a ; do echo -ne "$a\t"; grep -r --include='*.c' --include='*.h'  $a | wc -l; done | tee pci-defines-usage

	atomic_t
	cmpxchg
	circ_buf
	clamp_val
		clamp_t
	include/uapi/linux/types.h
		__sum16
	compiler-gcc.h
		__packed
								native_irq_return_iret
	_LINUX_SYSCALLS_H

	pthread_mutex_lock
		__pthread_mutex_lock
			__pthread_mutex_lock_full
				atomic_compare_and_exchange_val_acq
					__sync_val_compare_and_swap
						arch_atomic_val_compare_and_exchange libgcc
	kselftest

	random_fops
		random_write
			write_pool drivers/char/random.c
				arch_get_random_int arch/x86/include/asm/archrandom.h
					arch_has_random
					rdrand_int

	arch_get_random_longs
		rdrand_long

=== unsorted
{	unsorted
		levels
			virtual
			real/physical
		dma_cache_wback_inv

		panic

		communication

		usb_serial_driver : usb_driver

	IOMMU: buss address <-> phys address

	dma_pool
		dma_pool_create

	i2c_smbus_xfer

	i2c_algorithm smbus_algorithm
		.smbus_xfer	= vt596_access,
		.functionality	= vt596_func,

	netfilter_init
		nf_hooks

	nf_register_hooks
		nf_register_hook
			nf_hooks

	CONFIG_NETFILTER
	ipv4_netfilter_init
		nf_register_afinfo nf_ip_afinfo
			nf_afinfo
				nf_ip_route
					ip_route_output_key

	iptable_raw_init
		nf_register_hooks  ▻
	mmap_mem

	TIME time
	sys_gettimeofday
		do_gettimeofday
			xtime
			getnstimeofday
	sys_time
		get_seconds
			xtime_cache.tv_sec

	sys_times
		do_sys_times
		jiffies_64_to_clock_t(get_jiffies_64());
		jiffies_64
			jiffies

	sys_fstat64
	oprofile
		sys_lookup_dcookie		fs/dcookies.c

	sys_epoll_create		fs/eventpoll.c lkm2
		sys_epoll_create1
			anon_inode_getfd
				anon_inode_getfile
					d_alloc_pseudo
					d_alloc
					alloc_file lkm2
						get_empty_filp
						special_file
						file_take_write
						mnt_clone_write
						ima_counts_get

	sys_eventfd		fs/eventfd.c
		sys_eventfd2
			eventfd_file_create
				eventfd_fops
					eventfd_read
						eventfd_ctx_do_read
				anon_inode_getfile  ▻
			fd_install  ▻
}

== HW interfaces

{
	....
	5
		acpi_parse_mp_wake
			acpi_wake_cpu_handler_update
				&acpi_wakeup_cpu
		boot:
			native_cpu_up
				do_boot_cpu
					&start_secondary
					wakeup_secondary_cpu_64
						acpi_wakeup_cpu
							acpi_mp_wake_mailbox
							...
								start_secondary
					announce_cpu

		online:
				bringup_cpu
					__cpu_up
						...
							native_cpu_up?
					mtrr_ap_init


		secondary_startup_64_no_verify arch/x86/kernel/head_64.S
			start_secondary
				smp_store_cpu_info
					identify_secondary_cpu
						mtrr_ap_init
							mtrr_enabled
							&mtrr_rendezvous_handler
							set_mtrr_from_inactive_cpu
								stop_machine_from_inactive_cpu
									mtrr_rendezvous_handler
										mtrr_if
	generic_set_all ->
		generic_set_all
			prepare_set
			set_mtrr_state
			pat_init
			post_set


	set_mtrr
		stop_machine_cpuslocked
		stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);


=== arm aarch64
	{
	Hardware name: HPE ProLiant RL300 Gen11/ProLiant RL300 Gen11, BIOS 1.70 05/23/2024
	50.00 BogoMIPS
	5.14.0-284.67.1.el9_2.aarch64+64k
	(mockbuild@aarch64-024.brew-001.prod.us-east-1.aws.redhat.com)
	(gcc (GCC) 11.3.1 20221121 (Red Hat 11.3.1-4), GNU ld version 2.35.2-37.el9) #1 SMP PREEMPT_DYNAMIC Mon May 13 15:24:28 UTC 2024
	LPI Locality-specific Peripheral Interrupt
	AIC Apple Interrupt Controller
	GIC Generic Interrupt Controller
	SGI Software Generated Interrupt
	TGC Trusted Computing Group
	TPM Trusted Platform Module https://en.wikipedia.org/wiki/Trusted_Platform_Module
	PCR Platform Configuration Register

	rcu_spawn_one_boost_kthread
		rcub

	syscall_exit_to_user_mode
	irqentry_exit_to_user_mode
	...
	rseq_handle_notify_resume
		__rseq_handle_notify_resume
			rseq_update_cpu_id
				trace_rseq_update

	?
	__schedule
		rcu_note_context_switch
			"Start context switch"
			trace_rcu_utilization
				rcu_utilization
			rcu_qs
			rcu_tasks_qs(current, preempt);
			rcu_preempt_ctxt_queue
			trace_rcu_utilization(TPS("End context switch"));

	gic_irq_domain_alloc
		gic_irq_domain_map
			gic_chip
				gic_ipi_send_mask
					gic_send_sgi
	arch_send_call_function_single_ipi
		send_call_func_single_ipi
			native_send_call_func_single_ipi
				__apic_send_IPI
		smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
			ipi_raise
			__ipi_send_mask(ipi_desc[ipinr], target);
				chip->ipi_send_single(data, cpu);

	irq_create_mapping
	irq_create_mapping_affinity
		irq_create_mapping_affinity_locked
			irq_domain_alloc_descs ->

	gic_smp_init
		irq_domain_alloc_irqs
		__irq_domain_alloc_irqs
			irq_domain_alloc_descs
				__irq_alloc_descs ->
		set_smp_ipi_range
			ipi_desc
			request_percpu_irq
				__request_percpu_irq
					IRQF_PERCPU | IRQF_NO_SUSPEND
					irqaction
					__setup_irq->
			>ipi_handler
				do_handle_IPI
					[IPI_CALL_FUNC]		= "Function call interrupts",
					ipi_types
					trace_ipi_entry
						ipi_entry
					generic_smp_call_function_interrupt
						generic_smp_call_function_single_interrupt
							__flush_smp_call_function_queue ->

	__secondary_switched
	secondary_start_kernel
		cpu_startup_entry
		do_idle ->
			cpuidle_idle_call
				cpuidle_enter
					cpuidle_enter_state
						trace_cpu_idle_miss
			schedule_idle
				__schedule

	dnf install -y -q  bpftrace;  mount null -t tracefs /sys/kernel/tracing/
	bpftrace -e "kprobe:generic_handle_domain_irq { @[kstack()] = count();}"
	bpftrace -e "kprobe:generic_smp_call_function_single_interrupt { @[kstack()] = count();}"
	eval 'bpftrace -e "kprobe:$1 { @[kstack()] = count();}"'

	bpftrace -e "kprobe:set_cpus_allowed_common { @[kstack()] = count();}"

	sh -c 'sudo bpftrace -e "kprobe:$0 { @[kstack()] = count();}"' $ARG

	arch/arm64/kernel/entry.S
		generate_el1_vector
		entry_handler
		kernel_ventry
		el1h_64_irq

		el1h_64_irq_handler
			handle_arch_irq
			el1_interrupt
				__el1_irq
					do_interrupt_handler
						call_on_irq_stack
						gic_handle_irq ->
							generic_handle_domain_irq->
	gic_acpi_init
		gic_init_bases
			set_handle_irq
				handle_arch_irq
			> gic_handle_irq
				__gic_handle_irq_from_irqson
					__gic_handle_irq
						generic_handle_domain_irq->

	generic_handle_domain_irq
		irq_resolve_mapping
			__irq_resolve_mapping
		handle_irq_desc
			generic_handle_irq_desc ->

	generic_handle_domain_irq+48
		handle_percpu_devid_irq+140
			ipi_handler ->

	call_on_irq_stack
		gic_handle_irq
			__gic_handle_irq_from_irqson.isra.0
				generic_handle_domain_irq
					handle_percpu_devid_irq
						arch_timer_handler_phys
							hrtimer_interrupt
								hrtimer_wakeup

		crb_acpi_driver
			crb_acpi_add
				acpi_get_table
					acpi_tb_get_table
						acpi_tb_validate_table
							acpi_tb_acquire_table
								acpi_os_map_memory
									acpi_os_map_iomem
				acpi_table_tpm2
				start_method
				tpm_chip_register
					tpm_chip_start
						tpm_request_locality
						tpm_cmd_ready
						tpm_relinquish_locality
							crb_relinquish_locality
								__crb_relinquish_locality
									CRB_LOC_CTRL_RELINQUISH
									crb_wait_for_reg_32
					tpm_auto_startup
						tpm2_auto_startup
							tpm2_startup
							tpm2_do_selftest
								TPM2_CC_SELF_TEST
								tpm_transmit_cmd
							tpm2_get_cc_attrs_tbl
								tpm2_get_tpm_pt
									tpm_transmit_cmd ->
								tpm_buf_append_u32
									tpm_buf_append
					tpm_add_hwrng
						hwrng_register
	hwrng https://docs.kernel.org/admin-guide/hw_random.html

	el0t_64_sync
	el0t_64_sync_handler
	el0_svc
	do_el0_svc
	    el0_svc_common
		    invoke_syscall
			    __arm64_sys_read
				    ksys_read -> vfs_read -> rng_dev_read

			rng_dev_read
				reading_mutex
				get_current_rng
					get_current_rng_nolock

				rng_get_data ->

			/sys/devices/virtual/misc/hw_random/rng_current:tpm-rng-0
			devm_hwrng_register
				hwrng_register
			hwrng_manage_rngd
				hwrng_fillfn - thread
					?
					data_read
					rng_get_data
						tpm_hwrng_read
							tpm_get_random
								tpm2_get_random
								    tpm_transmit_cmd
									tpm_transmit
										bpftrace -e "kprobe:tpm_transmit { @[kstack()] = count();}"
										bpftrace -e 'kprobe:tpm_transmit { printf("%s \n",ksym(((struct tpm_chip *)arg0)->ops->send));}'
										tpm_try_transmit
											chip->ops->send
											crb_send len=12
											chip->ops->recv
											crb_recv
											tpm_calc_ordinal_duration
												tpm2_calc_ordinal_duration ->
											crb_cancel
												crb_do_acpi_start ->
											tpm_msleep ->
										tpm2_calc_ordinal_duration
											tpm2_ordinal_duration_index
										tpm_msleep
											usleep_range
												usleep_range_state
													schedule_hrtimeout_range
														schedule_hrtimeout_range_clock
												TASK_UNINTERRUPTIBLE
					add_hwgenerator_randomness
						credit_entropy_bits

	crb_send
		iowrite32
		memcpy_toio
			__memcpy_toio
				__raw_writeq
		wmb
		crb_do_acpi_start
			acpi_evaluate_dsm
			bpftrace -e "kprobe:acpi_evaluate_dsm { @[kstack()] = count();}"
		SMCCC System Monitor Call Convention
			https://developer.arm.com/Architectures/SMCCC
		SMC Secure Monitor Call
		priv->smc_func_id
		tpm_crb_smc_start
			arm_smccc_smc
				__arm_smccc_smc
					Perform the Secure Monitor Call
					arch/arm64/kernel/smccc-call.S
						SMCCC
							__arm_smccc_sve_check
	crb_recv
		CRB Command Response Buffer
		ioread32
		memcpy_fromio
		bpftrace -e 'kprobe:__memcpy_fromio { @start[tid] = nsecs;} kprobe:__memcpy_fromio /@start[tid] != 0/ {printf("%d ms\n",(nsecs - @start[tid])/1e3);delete(@start[tid]);}'
	 crb_cmd_ready
		__crb_cmd_ready
			crb_wait_for_reg_32
				ioread32

					hisi_rng_read
			add_hwgenerator_randomness

			taskset -cp 1 $(pgrep -f hwrng)
			ps -o psr,pid,comm -C hwrng

	for c in {10..12};do echo 0 > /sys/devices/system/cpu/cpu$c/online; echo 1 > /sys/devices/system/cpu/cpu$c/online; done
	for c in {16..127};do echo 0 > /sys/devices/system/cpu/cpu$c/online; done

	enum tpm2_pt_props {
		TPM2_PT_NONE = 0x00000000,
		TPM2_PT_GROUP = 0x00000100,
		TPM2_PT_FIXED = TPM2_PT_GROUP * 1,
		TPM2_PT_FAMILY_INDICATOR = TPM2_PT_FIXED + 0,
		TPM2_PT_LEVEL = TPM2_PT_FIXED + 1,
		TPM2_PT_REVISION = TPM2_PT_FIXED + 2,
		TPM2_PT_DAY_OF_YEAR = TPM2_PT_FIXED + 3,
		TPM2_PT_YEAR = TPM2_PT_FIXED + 4,
		TPM2_PT_MANUFACTURER = TPM2_PT_FIXED + 5,
		TPM2_PT_VENDOR_STRING_1 = TPM2_PT_FIXED + 6,
		TPM2_PT_VENDOR_STRING_2 = TPM2_PT_FIXED + 7,
		TPM2_PT_VENDOR_STRING_3 = TPM2_PT_FIXED + 8,
		TPM2_PT_VENDOR_STRING_4 = TPM2_PT_FIXED + 9,
		TPM2_PT_VENDOR_TPM_TYPE = TPM2_PT_FIXED + 10,
		TPM2_PT_FIRMWARE_VERSION_1 = TPM2_PT_FIXED + 11,
		TPM2_PT_FIRMWARE_VERSION_2 = TPM2_PT_FIXED + 12,
	}

	static check_ver(struct tpm_chip *chip)
	{
		// crb_acpi_add
		//	tpm_chip_register
		// 		tpm_add_hwrng
		u32 val1, val2;
		u64 version;

		tpm2_get_tpm_pt(chip, TPM2_PT_MANUFACTURER, &val, NULL);
		tpm2_get_tpm_pt(chip, TPM2_PT_FIRMWARE_VERSION_1, &val1, NULL);
		tpm2_get_tpm_pt(chip, TPM2_PT_FIRMWARE_VERSION_2, &val2, NULL);
		version = ((u64)val1 << 32) | val2;
	}



=== armada
	armada_370_pinctrl_driver
		"armada-370-pinctrl"
		"marvell,mv88f6710-pinctrl"
		armada_370_pinctrl_probe
			armada_370_pinctrl_info
			mvebu_pinctrl_probe
	LED
		gpio_led_init
		gpio_led_probe  ▻
		gpio_led_driver
			of_gpio_leds_match
				"gpio-leds"
			"leds-gpio"

				gpio_led_probe
					...
						gpio_blink_set
							led_dat->platform_gpio_blink_set
					create_gpio_led
						gpio_request
						gpio_direction_output
							CONFIG_DEBUG_GPIO
							chip->request
								mv_gpio_request
									mv_gpio_is_valid
							chip->direction_output
								mv_gpio_direction_output
					gpio_leds_create
						device_for_each_child_node
						fwnode_property_read_string

	arch_irq_work_raise
		apic->send_IPI_self(IRQ_WORK_VECTOR); //
			X86_TRAP_MF
			default_send_IPI_self
				__default_send_IPI_shortcut(APIC_DEST_SELF, vector, apic->dest_logical);
					__prepare_ICR(shortcut, vector, dest);
					native_apic_mem_write(APIC_ICR, cfg);
		... irq_work_interrupt

	platform_get_irq
		platform_get_resource
			resource_type

	CONFIG_SPI_SPIDEV
		spidev_init

	CONFIG_REGULATOR
	CONFIG_REGULATOR_FIXED_VOLTAGE
	regulator_fixed_voltage_init
		... reg_fixed_voltage_probe
			of_get_fixed_voltage_config
				of_get_regulator_init_data
					of_get_regulation_constraints
						of_get_property ... regulator-name

	CONFIG_OF
		of_unittest
			of_unittest_property_string
				of_property_match_string
				of_property_count_strings
				of_property_read_string
					of_find_property  ▻
				of_property_read_string_array
					of_property_read_string_helper
		of_property_read_*
			of_property_read_bool
				of_find_property
					__of_find_property
						of_prop_cmp
					properties
			of_property_read_u8
				of_property_read_u8_array
					of_find_property_value_of_size  ▻
			of_property_read_u32
				of_property_read_u32_array
					of_find_property_value_of_size
						of_find_property  ▻
					be32_to_cpup
						__be32_to_cpup
		fdt_translate_address
			count_cells
		of_match_bus
			of_busses
				count_cells
				of_bus_default_count_cells
					of_n_addr_cells
					of_n_size_cells

	 /proc/device-tree
	of_core_init
		for_each_of_allnodes
			for_each_of_allnodes_from
				__of_find_all_nodes
					of_root
					prev->child
					np->parent
					np->sibling
	of_unittest_platform_populate
		platform_device
		of_find_device_by_node
	of_parse_phandle
		__of_parse_phandle_with_args
	of_get_child_by_name
		for_each_child_of_node
			of_get_next_child
				__of_get_next_child
					next = next->sibling
					of_node_get(next))
						kobject_get
							kref_get
								atomic_inc_return
									atomic_add_return
										ATOMIC_OP_RETURN
											raw_local_irq_save
											op
											raw_local_irq_restore
					of_node_put(prev);
		of_node_cmp
	of_find_node_by_name
	of_get_child_count
	for_each_child_of_node
	of_node_put
		kobject_put
			kref_put
			kobject_release
				kobject_cleanup
					kobject_del
					t->release

	i2c_register_adapter
		of_i2c_register_devices
			of_i2c_register_device
				of_get_property
				be32_to_cpup

	ofdev->dev.of_node
		struct device_node *np = pdev->dev.of_node;
	of_get_gpio
		of_get_gpio_flags
			of_get_named_gpio_flags

	unflatten_dt_node
		device_type
	of_find_compatible_node
	of_device_is_compatible
		__of_device_is_compatible
			compat type name
				device_type
			__of_find_property
				of_prop_cmp
				np->properties
			of_prop_next_string
			of_compat_cmp
				strcasecmp
	ptr = of_get_property
		of_find_property  ▻
	if (ptr)
		value = be32_to_cpup(ptr);
	of_iomap
		of_address_to_resource
			of_get_address
	of_irq_to_resource
	irq_of_parse_and_map
		of_irq_map_one
			of_get_property  ▻
		irq_create_of_mapping
			irq_create_mapping
				irq_domain_associate
					irq_domain_associate_many
	DTC
	of_attach_node
	of_selftest
		device_node
		of_find_node_by_path
			allnodes
		of_selftest_parse_phandle_with_args
		of_selftest_property_match_string
	device_node
	dlpar_attach_node
		of_attach_node
			allnodes
		add_node_proc_entries
			proc_device_tree_add_node
	samples
		reg-init
			bcm87xx_of_reg_init
				of_get_property
				be32_to_cpup
			marvell_of_reg_init
				of_get_property
				be32_to_cpup
	i2c_new_client_device
	i2c_new_device
		i2c_bus_type
			i2c_device_match
				of_driver_match_device
					of_match_device  ▻
					of_match_table
				acpi_driver_match_device
				i2c_match_id
	platform_bus_init
		platform_bus_type
			platform_match
				of_driver_match_device
					of_match_device
						of_match_node
							__of_match_node
								__of_device_is_compatible
									of_compat_cmp
										strcasecmp
	CONFIG_PM
		dev_pm_ops
		platform_legacy_suspend
		platform_pm_suspend
		serial8250_suspend

		of_device_id

	PRCM	Power Reset and Clock Management

	PCS = Pinctrl Single
		pcs_probe
			devm_request_mem_region
			pcs_allocate_pin_table
				pcs_add_pin
					pcs_add_pin
			devm_ioremap  ▻
			pinctrl_register

	pinctrl CONFIG_PINCTRL
		mount null -t debugfs /sys/kernel/debug/
		head /sys/kernel/debug/pinctrl/*
		pinctrl_init
			pinctrl_init_debugfs
				pinctrl_ops (static var)
					pinctrl_open
						single_open
		        				seq_open
						pinctrl_show
							pinctrl_list
		pinctrl_register
			pinctrl_desc
				pinctrl_pin_desc
					PINCTRL_PIN
				pinctrl_ops (struct)
				pinmux_ops
				pinconf_ops

		pinctrl_request_gpio
			<- at91_gpio_request, tegra_gpio_request
			pinctrl_get_device_gpio_range
				pinctrl_match_gpio_range
			pinmux_request_gpio
				pin_request
					pin_desc_get
		devm_pinctrl_get_select_default	API
			device ▻  pinctrl
			device
			PINCTRL_STATE_DEFAULT
			PTR_ERR(dev->pins) == -EPROBE_DEFER
			is called from
				pinctrl
				serial_omap_probe omap2_mcspi_probe omap_hsmmc_probe omap_i2c_probe
			pinctrl
			devm_pinctrl_get_select
				devm_pinctrl_get
					devres_alloc
					pinctrl_get
						pinctrl_get_locked
							find_pinctrl
								pinctrl_list
								device  ▻ pinctrl
							create_pinctrl
								pinctrl_dt_to_map
								for_each_maps
									pinctrl_maps
									if (strcmp(map->dev_name, devname))
								add_setting
								pinctrl_list
				pinctrl_lookup_state
					pinctrl_lookup_state_locked
						find_state
							&p->states
						create_state
				pinctrl_select_state
		pinctrl_provide_dummies
			pinctrl_dummy_state = true

			pinctrl_register_mappings
			pinctrl_register_map
				< nhk8815_platform_init
				< mop500_href_family_pinmaps_init
				< dt_remember_or_free_map
				pinctrl_maps
			pinctrl_map

		samples

			at91_pinctrl_probe
				at91_pinctrl_probe_dt
					of_match_device
						of_match_node
							__of_match_node
								__of_device_is_compatible  ▻
				at91_pinctrl_desc
				platform_set_drvdata  ▻
				pinctrl_register
				pinctrl_add_gpio_range
					pinctrl_gpio_range
						gpio_ranges
				irq_chip gpio_irqchip
				irq_set_chip_and_handler

				at91_pinctrl_init	drivers/pinctrl/pinctrl-at91.c
					at91_gpio_driver
						at91_gpio_probe
							at91_gpio_template : gpio_chip
								at91_gpio_set
								at91_gpio_request
									pinctrl_request_gpio  ▻
							gpiochip_add  ▻
			u300_pmx_init
				u300_pmx_probe
					pinctrl_register
					u300_pmx_desc
					pinctrl_add_gpio_range  ▻
					u300_gpio_ranges
					U300_GPIO_RANGE
			u300_init_machine
				pinctrl_map
				u300_pinmux_map
					PIN_MAP_CONFIGS_PIN
				pinctrl_register_mappings  ▻

			u300_gpio_init
				u300_gpio_probe	drivers/pinctrl/pinctrl-coh901.c
					u300_gpio_chip : gpio_chip
						u300_gpio_request
							pinctrl_request_gpio  ▻
					gpiochip_add  ▻

			omap_gpio_drv_reg (init)
				omap_gpio_driver
					omap_gpio_probe
						omap_gpio_chip_init
							omap_gpio_request <- chip.request
								gpio_bank
								_set_gpio_triggering
								bank->base + bank->regs->pinctrl
							gpiochip_add  ▻

	serial_omap_init
		serial_omap_driver
			serial_omap_probe
				up->DTR_gpio = omap_up_info->DTR_gpio;
				gpio_request(omap_up_info->DTR_gpio, "omap-serial");

	depends on SERIAL_8250=m
	CONFIG_SERIAL_8250=m
	serial8250_init
		...
			serial8250_isa_init_ports
			serial8250_pops
				serial8250_config_port
					autoconfig
			serial8250_startup
				up->ier = UART_IER_RLSI | UART_IER_RDI;
				serial_port_out(port, UART_IER, up->ier);

			serial_link_irq_chain
				serial8250_interrupt
			serial8250_rx_chars
			serial8250_start_tx
				serial8250_tx_chars
			size_fifo
				serial_out
					io_serial_out
				serial_in
					io_serial_in
			serial8250_suspend
			stty
				tty_mode_ioctl
					TCGETS
						set_termios
							tmp_termios.c_ispeed = tty_termios_input_baud_rate(&tmp_termios);
							tmp_termios.c_ospeed = tty_termios_baud_rate(&tmp_termios);
							tty_set_termios
								set_termios
									? uart_set_termios
									serial8250_set_termios
										serial8250_do_set_termios
											uart_get_baud_rate
												tty_termios_baud_rate
													tty_termios_encode_baud_rate

	hid_init
		ret = bus_register(&hid_bus_type);
		hidraw_init

	hidg_touchscreen_dev_data

	// HID_MAIN_ITEM_TAG_END_COLLECTION
	hidg_dev_init
		platform_device_unregister(&hidg_dev)
			hidg_dev
				.name			= "hidg",
				.dev.platform_data	= &hidg_dev_data,
					hidg_func_descriptor

	mknod /dev/hidg0 c 254 0
	hid_gadget_test /dev/hidg0 keyboard
	hidg_init	drivers/usb/gadget/hid.c
		platform_driver_probe(&hidg_plat_driver, hidg_plat_driver_probe);
			hidg_plat_driver_probe
				struct hidg_func_descriptor *func = pdev->dev.platform_data;
				list_add_tail(&entry->node, &hidg_func_list);
		usb_composite_register
			usb_gadget_register_driver	drivers/usb/musb/musb_gadget.c
				... bind
					composite_bind
					composite_unbind
		hid_bind
			ghid_setup
			usb_add_config(cdev, &config_driver);
				config_driver
					struct usb_device_descriptor device_desc
					do_config	drivers/usb/gadget/hid.c
						hidg_bind_config	drivers/usb/gadget/f_hid.c

							hidg_desc	: hid_descriptor
							.bInterfaceClass	= USB_CLASS_HID,
							hidg->bInterfaceProtocol = fdesc->protocol;
							hidg->report_desc = kmemdup(fdesc->report_desc,
							status = usb_add_function(c, &hidg->func);

							...
								hidg_bind
									hidg_fs_descriptors
										hidg_interface_desc : usb_interface_descriptor
										hidg_interface_desc.iInterface = status;
										hidg_hs_in_ep_desc
									hidg_hs_descriptors
			ghid_setup	drivers/usb/gadget/f_hid.c
			usb_add_config

	gpio_request
	gpio_request_one
	pca953x_probe
		gpiochip_add
			gpio_chip
			of_gpiochip_add
			gpiochip_export
				gpiochip_add_export_v2
	alias_export_store
		gpio_map_name_to_num

	iomem  I/O Memory

		 /proc/iomem

		./include/linux/ioport.h

		check_mem_region
		request_mem_region	__request_region
		release_mem_region

		./kernel/resource.c

		resource
		ioresources_init
			proc_ioports_operations
				ioport_resource	/proc/ioports
			proc_iomem_operations
				iomem_resource	/proc/iomem
		ioremap
			__arm_ioremap
				arch_ioremap_caller
					__arm_ioremap_caller
						__arm_ioremap_pfn_caller
			!CONFIG_MMU return (void __iomem*) (unsigned long)offset;

			? __ioremap
				__pgprot
				ioremap_page_range
					pgd_addr_end
					ioremap_pud_range
						pud_alloc
						pud_addr_end
						ioremap_pmd_range
							ioremap_pte_range
								set_pte_at
			__ioremap
				get_vm_area
					__get_vm_area
				remap_area_pages
					pmd_alloc
						__pmd_alloc
							pgd_present
						pmd_offset
					remap_area_pmd
						pte_alloc_kernel
						remap_area_pte
							set_pte
					flush_cache_vmap
			x86?
			ioremap_nocache
				__ioremap_caller
				ioremap_page_range  ▻

		I/O Ports

		/proc/ioports

		depricated check_region

		request_region
			__request_region kernel/resource.c
				__request_resource
		release_region

		resource_op

		void release_region(unsigned long start, unsigned long len);
		resource

		linux/ioport.h

		kernel/resource.c

		==> ./oo/comm/reg/notes.txt <==
		Chapter 8
		Hardware Management
		I/O Ports
		I/O Memory
		/proc/ioports

=== HW busses

==== SPI

			spi_register_board_info
				spi_board_info
				spi_match_master_to_boardinfo
					spi_new_device
						spi_alloc_device
						spi_add_device

			API
				SPI_IOC_MESSAGE
				spi_ioc_transfer
			spi_dev_init
				spidev_fops
					spidev_open
						spi_dev_get_by_minor
							spi_dev_array
						spi_get_adapter
							adapters
						filp->private_data = spidev;
					spidev_write ( file )
						spi_write ( spi_client )
							spi_transfer ( spi_adapter )
								-> spi_davinci_xfer  ( spi_adapter )
					spidev_ioctl  ▻

			spi_client
				-> spi_adapter
					spi_algorithm
			spi_davinci_device
				spi_adapter
					-> davinci_spi_algo
						spi_davinci_xfer
							spi_davinci_readbytes
								spi_davinci_cmd_complete
								davinci_spiConfig
							spi_davinci_sendbytes
								spiregs
									SPIDAT0
									SPIDAT1
						-> spi_davinci_hw_init

			spi_davinci_init
				spi_davinci_saved_oadap : spi_davinci_device
				spi_davinci_dev->regs = (spiregsovly)DAVINCI_SPI_BASE_ADDR;
			if (spi_add_adapter(adap) < 0)

			spidev_ioctl SPI_IOCTL_SET_CONFIG
				spi_control
					algo_control
						spi_davinci_control SPI_IOCTL_SET_CONFIG
							spi_davinci_hw_init
								spi_davinci_hwConfig
				spidev_message
					spidev_sync
						spi_async
							__spi_async

==== i2c
		common
			i2c_client
			i2c_master_send

			i2c_add_driver
				i2c_driver

		i2c-davinci
			i2c-davinci.c
				i2c_davinci_isr
					icdrr

			i2cdev_write
				i2c_master_send
			i2cdev_read
				i2c_master_recv
			i2cdev_ioctl i2c-dev.c
				I2C_SLAVE
					client->addr = arg;
				i2c_transfer
				I2C_RDWR
				I2C_SMBUS
					i2c_smbus_xfer

			i2c_smbus_read_byte_data
				i2c_smbus_xfer
					smbus_xfer
					i2c_smbus_xfer_emulated

			pcf_work

			i2c-core.c
				i2c_transfer
					master_xfer
						i2c_davinci_xfer
							i2c_davinci_xfer_msg
								davinci_i2cregs
				i2c_master_send
					master_xfer
				i2c_master_recv
					I2C_M_RD
					master_xfer

			i2c_adapter
				i2c_davinci_device
					davinci_i2cregsovly
						davinci_i2cregs

			i2c_davinci_init
				i2c_davinci_dev.regs = (davinci_i2cregsovly)I2C_BASE;
					IO_ADDRESS(DAVINCI_I2C_BASE)
						DAVINCI_I2C_BASE		   (0x01C21000)
						io_p2v(pa)   (((pa) & (IO_SIZE-1)) + IO_VIRT)
							#define IO_PHYS	     0x01c00000
							#define IO_VIRT	     0xe1000000
							#define IO_SIZE	     0x00400000

==== USB

		module_usb_driver
			usb_register
				usb_register_driver
		libusb
			http://www.libusb.org/
			http://libusb.sourceforge.net/api-1.0/index.html
			/mnt/1TB/main/tests/iboard/usbtouch-irq.c
			libusb_open_device_with_vid_pid
		ehci_hcd
			ehci_hcd_init
				PLATFORM_DRIVER
					ehci_hcd_omap_driver
						ehci_hcd_omap_probe
							regulator_get
								_regulator_get
									create_regulator
							usb_create_hcd
							ehci_omap_hc_driver
							usb_add_hcd  ▻
				ehci_pci_driver
					usb_hcd_pci_probe  ▻
					ehci_pci_hc_driver
						urb_enqueue ehci_urb_enqueue
						irq ehci_irq
		ohci_hcd
			ohci_hcd_pci_init
				pci_register_driver ohci_pci_driver	pci_driver
					ohci_pci_hc_driver
						ohci_irq
						urb_enqueue ohci_urb_enqueue
					usb_hcd_pci_probe
						dev->irq
						pci_enable_device  ▻
							...
								pirq_enable_irq
						usb_create_hcd
							usb_bus_init
						pci_resource_start (dev, 0);
						pci_resource_len (dev, 0);
						request_mem_region
						pci_set_master
							pcibios_set_master
						usb_add_hcd
							usb_register_bus
								usb_host_class
								class_device_create   ▻
							usb_alloc_dev usb_hcd.usb_bus
							usb_hcd_irq
								hcd->driver->irq hc_driver  irq
									ohci_irq
										ohci_readl
										ohci_writel
									ehci_irq  ▻
							register_root_hub
							usb_hcd_request_irqs
								request_irq(irqnum, &usb_hcd_irq
		device_reprobe
		usb_hcd
			.usb_bus
				device *controller
			hc_driver
		generic
			usb_exit
			usb_init drivers/usb/core/usb.c
				bus_register
					subsystem_register
					bus_type  ▻
				usb_bus_type
					usb_device_match
					usb_hotplug
					usb_generic_suspend
					usb_generic_resume
				usb_register(&usbfs_driver);
					usb_register_driver
						driver_register  ▻
				usbdev_init
					register_chrdev_region(USB_DEVICE_DEV, USB_DEVICE_MAX, "usb_device");
					cdev_init(&usb_device_cdev, &usbfs_device_file_operations);
					retval = cdev_add(&usb_device_cdev, USB_DEVICE_DEV, USB_DEVICE_MAX);
					usb_device_class = class_create(THIS_MODULE, "usb_device");
					usb_register_notify
						blocking_notifier_chain_register(&usb_notifier_list, nb);
							notifier_chain_register
								rcu_assign_pointer ????

				usbfs_init
					usb_fs_type
				usb_hub_init
				usb_register_device_driver(&usb_generic_driver, THIS_MODULE);

			usb_driver
				probe usb_interface
				usbdrv_wrap
			usb_device_driver
				usbdrv_wrap
					device_driver

			usb_bulk_msg
				usb_fill_bulk_urb
				usb_start_wait_urb

			"send_packet"
				usb_submit_urb  ▻
				usb_fill_int_urb  / usb_fill_bulk_urb
				wait_for_completion

			irda_usb_open

			kernel.lnk/build/drivers/usb/serial/ir-usb.c
				ir_init
					usb_serial_register
						ir_device : usb_serial_driver
							ir_write
						usb_serial_bus_register
					usb_register

			usb_set_interface

			checkintf claimintf usb_driver_claim_interface
			usbdev_release releaseintf usb_driver_release_interface

			libusb_claim_interface
				op_claim_interface
					detach_kernel_driver_and_claim
						IOCTL_USBFS_DISCONNECT_CLAIM
						op_detach_kernel_driver
							IOCTL_USBFS_DISCONNECT
						claim_interface
						(kernel)
						proc_disconnect_claim
							claimintf
					claim_interface
						IOCTL_USBFS_CLAIMINTF

			usb_reset_configuration
				usb_disable_endpoint
				usb_disable_device

			hub_port_connect_change
				usb_new_device

			usbfs_devices_fops
				usb_device_read
					usb_device_dump
						usb_dump_desc
							usb_dump_device_descriptor
							usb_dump_device_strings
							usb_dump_config
								usb_dump_config_descriptor
								usb_dump_interface
									usb_dump_endpoint_descriptor

			mon_init
				debugfs_create_dir
					debugfs_create_file
						debugfs_create_by_name
				... debugfs_remove
				mon_bus_init
					mon_fops_text
						mon_text_open
							rnf_complete = mon_text_complete;
							mon_reader_add
			usb_hcd_giveback_urb
			ehci_irq
				ehci_work
					scan_async
						qh_completions
							? ehci_urb_done
								usb_hcd_giveback_urb hcd.c printk RCV
									usbmon_urb_complete ( mon_reader_add )
										urb_complete
											mon_complete
												rnf_complete
													mon_text_event
														mon_text_get_setup
														mon_text_get_data
									usb_api_blocking_completion
										complete((struct completion *)urb->context);
									-> wait_for_completion
								usb_put_urb

			ehci_urb_enqueue
				submit_async

			usb_skel_init
				usb_register
					.skel_driver
						skel_probe
							usb_register_dev
								.skel_class
									skel_fops  ▻
								class_device_create  ▻

			skel_fops
				skel_open
					usb_find_interface
						__find_interface
							driver_for_each_device
					usb_autopm_get_interface
						pm_runtime_get_sync ->
				skel_write

					usb_alloc_urb
						kmalloc
						usb_init_urb
					usb_fill_bulk_urb
					usb_submit_urb printk SBM
						usb_hcd_submit_urb
							hcd->driver->urb_enqueue
						{ obsolete
						submit_urb
							hcd_submit_urb
								usbmon_urb_submit
								urb_enqueue
									uhci_urb_enqueue
										uhci_submit_control
										uhci_submit_interrupt
											uhci_submit_common
												uhci_append_queued_urb
										uhci_submit_bulk
											uhci_submit_common
											uhci_inc_fsbr
							hcd->driver->urb_enqueue
						}
			usbfs_nb

			usbfs_add_bus
				create_special_files
					usbfs_devices_fops  ▻

			usb_reset_device
			hub_port_init
				usb_get_device_descriptor
					usb_get_descriptor
						usb_control_msg
							usb_internal_control_msg
								usb_fill_control_urb
								usb_start_wait_urb
									usb_submit_urb  ▻
									wait_for_completion  ▻

			uhci_hcd_init
			uhci_irq
				uhci_scan_schedule
					uhci_transfer_result
						uhci_result_control
						uhci_result_common
					uhci_finish_completion
						uhci_finish_urb
							uhci_destroy_urb_priv
							usb_hcd_giveback_urb  ▻
			usbdev_add
				class_device_create usb_device_class

CKI Continuous Kernel Integration


	rcu_spawn_cpu_nocb_kthread
		kthread_run
		rcu_nocb_gp_kthread
			nocb_gp_wait

host 			411us
normal	rtla timerlat	50ms
rt	rtla timerlat	5649us
	offline 	3420us


tp
	725us


(sleep 5; echo 0 > /sys/devices/system/cpu/cpu3/online) & rtla timerlat top -c 1

echo 1 > /sys/devices/system/cpu/cpu3/online;(sleep 5; echo QQQ; echo 0 > /sys/devices/system/cpu/cpu3/online) & rtla timerlat hist -b1000 -E 10 -c 1 -a 3000|


patchpal/
	gui/

		main
		
		PatchpalApplication

		src/patchpal/main.rs
			start
		DiffModel
			run_ai
		command_ai
			run_ai
				call_ai_rpc
					patchpal.usersys.redhat.com:9000
			parse_result
	backend/
		./src/patchpal/inference/chatbot.py
			main
				WrappedTokenizer
					infer_chatbot
		inference/
			server.py
				main
					WrappedTokenizer
					ThreadingHTTPServer
					httpd.serve_forever
					
                InferHandler
                    do_POST
                        infer
			test_server.py
				InferClient
		train/
			src/patchpal/train/metrics.py
				main
					postprocess_results
					save_results_postprocessed
					save_metrics_to_file
					plot_and_save_distribution
					all_roc_plots
                    all_distribution_plots
		train
			src/patchpal/train/train.py
				main
					WrappedTokenizer
					WrappedModel
		main
			parse_args
			WrappedTokenizer
			AutoTokenizer
			WrappedModel
				train
				Trainer
				TrainingArguments
				save_model


perf probe --add call_timer_fn
perf record -g -e probe:call_timer_fn -aR sleep 1
perf report

perf record --overwrite -a -e cycles --switch-output=signal
perf script -i perf.data...

gcc msr-tools/rdmsr.c -o /usr/local/bin/rdmsr


rdmsrq_safe_on_cpu
	rdmsr_safe_on_cpu
msr_fop
	smsr_read
		rdmsr_safe_on_cpu
			INIT_CSD
			__rdmsr_safe_on_cpu
				rdmsr_safe
			smp_call_function_single_async
			wait_for_completion


check owner 


sys_uselib

{
  "12345": {
    "command": "",
    "args": [],
    "env": {},
    "working_directory": null,
    "start_on_launch": true
  }
}


git commit --fixup=$(git rev-list main --grep="kpm_mcp.py with tests")

nohz_balancer_kick
	kick_ilb
		find_new_ilb
		smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);

ftrace_graph_ret_addr
arch/x86/kernel/ftrace_64.S
	return_to_handler

TPMI
	drivers/platform/x86/intel/tpmi_power_domains.h
		tpmi_init
		tpmi_get_punit_core_number
	drivers/platform/x86/intel/tpmi_power_domains.c
	drivers/platform/x86/intel/vsec_tpmi.c
		tpmi_get_resource_count
		intel_vsec_tpmi_init


git remote add $W git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-$W.git
git worktree list
git fetch $W
git -C ~/linux/main worktree add ../$W $W/for-next

for kdiff3
	~/cmnd/log-preprocess.py
	^1,2,3:x

dconf write /org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom2/command '"/home/costa/cmnd/key.sh record"'
dconf write /org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom1/command '"/home/costa/cmnd/key.sh stop"'

dconf dump /org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/


p=$(git format-patch -v1 -5 --cover-letter --cover-from-description=subject)
checkpatch.pl $p
m=$(get_maintainer.pl --norolestats --separator=, $p)
git send-email --to="$m" $p

Commit message structure:
	Current state, a problem, issue or bug
	Rationale
	Implementation
	Impact

since 5.15
grep schedule_rtlock /proc/kallsyms
grep -R schedule_rtmutex /proc/kallsyms
cat /sys/kernel/realtime

input_set_capability

Python trick
	def key_values(fields):
	    """ Extract key:value pairs from a list of fields and return them as a dictionary """
	    return { k: v for f in fields if ':' in f for k, v in [f.split(":", 1)] if k and v }



timerlat hist -D -c 1-3
Error parsing the cpu set 1-3
Invalid -c cpu list
costa-tp ➜  tracing/rtla > L
timerlat hist --duration 1  -E 10 -D -c 1,3  -a 100 
Error parsing the cpu set 1,3
Invalid -c cpu list


MPAM - Arm Memory System Resource Partitioning and Monitoring


=(GOOGLEFINANCE(c6) / IFERROR(INDEX(GOOGLEFINANCE(c6, "price", TODAY()-1), 2, 2), IFERROR(INDEX(GOOGLEFINANCE(c6, "price", TODAY()-2), 2, 2), INDEX(GOOGLEFINANCE(c6, "price", TODAY()-3), 2, 2)))) - 1


=((GOOGLEFINANCE(c9,"price")*GOOGLEFINANCE(d9)) / (IFERROR(INDEX(GOOGLEFINANCE(c9,"price",TODAY()-7),2,2),INDEX(GOOGLEFINANCE(c9,"price",TODAY()-9),2,2)) * IFERROR(INDEX(GOOGLEFINANCE(d9,"price",TODAY()-7),2,2),INDEX(GOOGLEFINANCE(d9,"price",TODAY()-9),2,2))))-1


=((GOOGLEFINANCE(C2,"price")*GOOGLEFINANCE(D2)) / (IFERROR(INDEX(GOOGLEFINANCE(C2,"price",TODAY()-7),2,2),INDEX(GOOGLEFINANCE(C2,"price",TODAY()-9),2,2)) * IFERROR(INDEX(GOOGLEFINANCE(D2,"price",TODAY()-7),2,2),INDEX(GOOGLEFINANCE(D2,"price",TODAY()-9),2,2))))-1




=(googlefinance(b32)/googlefinance(c32) / index(googlefinance(b32, "price", today()-7), 2, 2) * 
 iferror(
   index(googlefinance(c32, "price", today()-7), 2, 2), 
   iferror(
     index(googlefinance(c32, "price", today()-8), 2, 2), 
     index(googlefinance(c32, "price", today()-9), 2, 2)
   )
 ))-1


=(
  googlefinance(b32) / googlefinance(c32) / 
  iferror(
    index(googlefinance(b32, "price", today()-30), 2, 2), 
    iferror(
      index(googlefinance(b32, "price", today()-31), 2, 2), 
      index(googlefinance(b32, "price", today()-32), 2, 2)
    )
  ) * iferror(
    index(googlefinance(c32, "price", today()-30), 2, 2), 
    iferror(
      index(googlefinance(c32, "price", today()-31), 2, 2), 
      index(googlefinance(c32, "price", today()-32), 2, 2)
    )
  )
)-1


PPA
ITA
XAR


https://lore.kernel.org/linux-rt-devel/216light.css
/*
 * Underlined links add visual noise which make them hard-to-read.
 * Use colors to make them stand out, instead.
 */
a:link { color:#00f; text-decoration:none }

nvim
	~/.config/nvim/
	~/.config/nvim/init.lua
	~/.config/nvim/after
	~/.config/nvim/after/ftplugin
	~/.config/nvim/after/syntax
	~/.config/nvim/ini.vim
	~/.config/nvim/lua
	~/.config/nvim/lua/quickfix_colors.lua
	~/.config/nvim/lua/smart_grep.lua
	~/.config/nvim/lua/smart_grep_async.lua



add to map:
	dev_get_drvdata     │ device model         │
	netdev_priv         │ networking           │
	seq_printf          │ procfs/seq           │
	devm_kzalloc        │ device resource mgmt │
	atomic_read         │ atomics              │
	rcu_read_unlock     │ RCU                  │
	rcu_read_lock       │ RCU                  │
	sysfs_emit          │ sysfs                │
	module_init         │ modules              │
	copy_to_user        │ user access          │
	kfree_skb           │ networking           │
	free_irq            │ interrupts           │
	dma_alloc_coherent  │ DMA                  │
	debugfs_create_file │ debugfs