* remotes/origin/tmp-2f0de51: Linux 4.4.38 esp6: Fix integrity verification when ESN are used esp4: Fix integrity verification when ESN are used ipv4: Set skb->protocol properly for local output ipv6: Set skb->protocol properly for local output Don't feed anything but regular iovec's to blk_rq_map_user_iov constify iov_iter_count() and iter_is_iovec() sparc64: fix compile warning section mismatch in find_node() sparc64: Fix find_node warning if numa node cannot be found sparc32: Fix inverted invalid_frame_pointer checks on sigreturns net: ping: check minimum size on ICMP header length net: avoid signed overflows for SO_{SND|RCV}BUFFORCE geneve: avoid use-after-free of skb->data sh_eth: remove unchecked interrupts for RZ/A1 net: bcmgenet: Utilize correct struct device for all DMA operations packet: fix race condition in packet_set_ring net/dccp: fix use-after-free in dccp_invalid_packet netlink: Do not schedule work from sk_destruct netlink: Call cb->done from a worker thread net/sched: pedit: make sure that offset is valid net, sched: respect rcu grace period on cls destruction net: dsa: bcm_sf2: Ensure we re-negotiate EEE during after link change l2tp: fix racy SOCK_ZAPPED flag check in l2tp_ip{,6}_bind() rtnetlink: fix FDB size computation af_unix: conditionally use freezable blocking calls in read net: sky2: Fix shutdown crash ip6_tunnel: disable caching when the traffic class is inherited net: check dead netns for peernet2id_alloc() virtio-net: add a missing synchronize_net() Linux 4.4.37 arm64: suspend: Reconfigure PSTATE after resume from idle arm64: mm: Set PSTATE.PAN from the cpu_enable_pan() call arm64: cpufeature: Schedule enable() calls instead of calling them via IPI pwm: Fix device reference leak mwifiex: printk() overflow with 32-byte SSIDs PCI: Set Read Completion Boundary to 128 iff Root Port supports it (_HPX) PCI: Export pcie_find_root_port rcu: Fix soft lockup for rcu_nocb_kthread ALSA: pcm : Call kill_fasync() in stream lock x86/traps: Ignore high word of regs->cs in early_fixup_exception() kasan: update kasan_global for gcc 7 zram: fix unbalanced idr management at hot removal ARC: Don't use "+l" inline asm constraint Linux 4.4.36 scsi: mpt3sas: Unblock device after controller reset flow_dissect: call init_default_flow_dissectors() earlier mei: fix return value on disconnection mei: me: fix place for kaby point device ids. mei: me: disable driver on SPT SPS firmware drm/radeon: Ensure vblank interrupt is enabled on DPMS transition to on mpi: Fix NULL ptr dereference in mpi_powm() [ver #3] parisc: Also flush data TLB in flush_icache_page_asm parisc: Fix race in pci-dma.c parisc: Fix races in parisc_setup_cache_timing() NFSv4.x: hide array-bounds warning apparmor: fix change_hat not finding hat after policy replacement cfg80211: limit scan results cache size tile: avoid using clocksource_cyc2ns with absolute cycle count scsi: mpt3sas: Fix secure erase premature termination Fix USB CB/CBI storage devices with CONFIG_VMAP_STACK=y USB: serial: ftdi_sio: add support for TI CC3200 LaunchPad USB: serial: cp210x: add ID for the Zone DPMX usb: chipidea: move the lock initialization to core file KVM: x86: check for pic and ioapic presence before use KVM: x86: drop error recovery in em_jmp_far and em_ret_far iommu/vt-d: Fix IOMMU lookup for SR-IOV Virtual Functions iommu/vt-d: Fix PASID table allocation sched: tune: Fix lacking spinlock initialization UPSTREAM: trace: Update documentation for mono, mono_raw and boot clock UPSTREAM: trace: Add an option for boot clock as trace clock UPSTREAM: timekeeping: Add a fast and NMI safe boot clock ANDROID: goldfish_pipe: fix allmodconfig build ANDROID: goldfish: goldfish_pipe: fix locking errors ANDROID: video: goldfishfb: fix platform_no_drv_owner.cocci warnings ANDROID: goldfish_pipe: fix call_kern.cocci warnings arm64: rename ranchu defconfig to ranchu64 ANDROID: arch: x86: disable pic for Android toolchain ANDROID: goldfish_pipe: An implementation of more parallel pipe ANDROID: goldfish_pipe: bugfixes and performance improvements. ANDROID: goldfish: Add goldfish sync driver ANDROID: goldfish: add ranchu defconfigs ANDROID: goldfish_audio: Clear audio read buffer status after each read ANDROID: goldfish_events: no extra EV_SYN; register goldfish ANDROID: goldfish_fb: Set pixclock = 0 ANDROID: goldfish: Enable ACPI-based enumeration for goldfish audio ANDROID: goldfish: Enable ACPI-based enumeration for goldfish framebuffer ANDROID: video: goldfishfb: add devicetree bindings BACKPORT: staging: goldfish: audio: fix compiliation on arm BACKPORT: Input: goldfish_events - enable ACPI-based enumeration for goldfish events BACKPORT: goldfish: Enable ACPI-based enumeration for goldfish battery BACKPORT: drivers: tty: goldfish: Add device tree bindings BACKPORT: tty: goldfish: support platform_device with id -1 BACKPORT: Input: goldfish_events - add devicetree bindings BACKPORT: power: goldfish_battery: add devicetree bindings BACKPORT: staging: goldfish: audio: add devicetree bindings ANDROID: usb: gadget: function: cleanup: Add blank line after declaration cpufreq: sched: Fix kernel crash on accessing sysfs file usb: gadget: f_mtp: simplify ptp NULL pointer check cgroup: replace unified-hierarchy.txt with a proper cgroup v2 documentation cgroup: rename Documentation/cgroups/ to Documentation/cgroup-legacy/ cgroup: replace __DEVEL__sane_behavior with cgroup2 fs type writeback: initialize inode members that track writeback history mm: page_alloc: generalize the dirty balance reserve block: fix module reference leak on put_disk() call for cgroups throttle Linux 4.4.35 netfilter: nft_dynset: fix element timeout for HZ != 1000 IB/cm: Mark stale CM id's whenever the mad agent was unregistered IB/uverbs: Fix leak of XRC target QPs IB/core: Avoid unsigned int overflow in sg_alloc_table IB/mlx5: Fix fatal error dispatching IB/mlx5: Use cache line size to select CQE stride IB/mlx4: Fix create CQ error flow IB/mlx4: Check gid_index return value PM / sleep: don't suspend parent when async child suspend_{noirq, late} fails PM / sleep: fix device reference leak in test_suspend uwb: fix device reference leaks mfd: core: Fix device reference leak in mfd_clone_cell iwlwifi: pcie: fix SPLC structure parsing rtc: omap: Fix selecting external osc clk: mmp: mmp2: fix return value check in mmp2_clk_init() clk: mmp: pxa168: fix return value check in pxa168_clk_init() clk: mmp: pxa910: fix return value check in pxa910_clk_init() drm/amdgpu: Attach exclusive fence to prime exported bo's. (v5) crypto: caam - do not register AES-XTS mode on LP units ext4: sanity check the block and cluster size at mount time kbuild: Steal gcc's pie from the very beginning x86/kexec: add -fno-PIE scripts/has-stack-protector: add -fno-PIE kbuild: add -fno-PIE i2c: mux: fix up dependencies can: bcm: fix warning in bcm_connect/proc_register mfd: intel-lpss: Do not put device in reset state on suspend fuse: fix fuse_write_end() if zero bytes were copied KVM: Disable irq while unregistering user notifier KVM: x86: fix missed SRCU usage in kvm_lapic_set_vapic_addr x86/cpu/AMD: Fix cpu_llc_id for AMD Fam17h systems Linux 4.4.34 sparc64: Delete now unused user copy fixup functions. sparc64: Delete now unused user copy assembler helpers. sparc64: Convert U3copy_{from,to}_user to accurate exception reporting. sparc64: Convert NG2copy_{from,to}_user to accurate exception reporting. sparc64: Convert NGcopy_{from,to}_user to accurate exception reporting. sparc64: Convert NG4copy_{from,to}_user to accurate exception reporting. sparc64: Convert U1copy_{from,to}_user to accurate exception reporting. sparc64: Convert GENcopy_{from,to}_user to accurate exception reporting. sparc64: Convert copy_in_user to accurate exception reporting. sparc64: Prepare to move to more saner user copy exception handling. sparc64: Delete __ret_efault. sparc64: Handle extremely large kernel TLB range flushes more gracefully. sparc64: Fix illegal relative branches in hypervisor patched TLB cross-call code. sparc64: Fix instruction count in comment for __hypervisor_flush_tlb_pending. sparc64: Fix illegal relative branches in hypervisor patched TLB code. sparc64: Handle extremely large kernel TSB range flushes sanely. sparc: Handle negative offsets in arch_jump_label_transform sparc64 mm: Fix base TSB sizing when hugetlb pages are used sparc: serial: sunhv: fix a double lock bug sparc: Don't leak context bits into thread->fault_address tty: Prevent ldisc drivers from re-using stale tty fields tcp: take care of truncations done by sk_filter() ipv4: use new_gw for redirect neigh lookup net: __skb_flow_dissect() must cap its return value sock: fix sendmmsg for partial sendmsg fib_trie: Correct /proc/net/route off by one error sctp: assign assoc_id earlier in __sctp_connect ipv6: dccp: add missing bind_conflict to dccp_ipv6_mapped ipv6: dccp: fix out of bound access in dccp_v6_err() dccp: fix out of bound access in dccp_v4_err() dccp: do not send reset to already closed sockets tcp: fix potential memory corruption ip6_tunnel: Clear IP6CB in ip6tunnel_xmit() bgmac: stop clearing DMA receive control register right after it is set net: mangle zero checksum in skb_checksum_help() net: clear sk_err_soft in sk_clone_lock() dctcp: avoid bogus doubling of cwnd after loss ARM: 8485/1: cpuidle: remove cpu parameter from the cpuidle_ops suspend hook Linux 4.4.33 netfilter: fix namespace handling in nf_log_proc_dostring btrfs: qgroup: Prevent qgroup->reserved from going subzero mmc: mxs: Initialize the spinlock prior to using it ASoC: sun4i-codec: return error code instead of NULL when create_card fails ACPI / APEI: Fix incorrect return value of ghes_proc() i40e: fix call of ndo_dflt_bridge_getlink() hwrng: core - Don't use a stack buffer in add_early_randomness() lib/genalloc.c: start search from start of chunk mei: bus: fix received data size check in NFC fixup iommu/vt-d: Fix dead-locks in disable_dmar_iommu() path iommu/amd: Free domain id when free a domain of struct dma_ops_domain tty/serial: at91: fix hardware handshake on Atmel platforms dmaengine: at_xdmac: fix spurious flag status for mem2mem transfers drm/i915: Respect alternate_ddc_pin for all DDI ports KVM: MIPS: Precalculate MMIO load resume PC scsi: mpt3sas: Fix for block device of raid exists even after deleting raid disk scsi: qla2xxx: Fix scsi scan hang triggered if adapter fails during init iio: orientation: hid-sensor-rotation: Add PM function (fix non working driver) iio: hid-sensors: Increase the precision of scale to fix wrong reading interpretation. clk: qoriq: Don't allow CPU clocks higher than starting value toshiba-wmi: Fix loading the driver on non Toshiba laptops drbd: Fix kernel_sendmsg() usage - potential NULL deref usb: gadget: u_ether: remove interrupt throttling USB: cdc-acm: fix TIOCMIWAIT staging: nvec: remove managed resource from PS2 driver Revert "staging: nvec: ps2: change serio type to passthrough" drivers: staging: nvec: remove bogus reset command for PS/2 interface staging: iio: ad5933: avoid uninitialized variable in error case pinctrl: cherryview: Prevent possible interrupt storm on resume pinctrl: cherryview: Serialize register access in suspend/resume ARC: timer: rtc: implement read loop in "C" vs. inline asm s390/hypfs: Use get_free_page() instead of kmalloc to ensure page alignment coredump: fix unfreezable coredumping task swapfile: fix memory corruption via malformed swapfile dib0700: fix nec repeat handling ASoC: cs4270: fix DAPM stream name mismatch ALSA: info: Limit the proc text input size ALSA: info: Return error for invalid read/write arm64: Enable KPROBES/HIBERNATION/CORESIGHT in defconfig arm64: kvm: allows kvm cpu hotplug arm64: KVM: Register CPU notifiers when the kernel runs at HYP arm64: KVM: Skip HYP setup when already running in HYP arm64: hyp/kvm: Make hyp-stub reject kvm_call_hyp() arm64: hyp/kvm: Make hyp-stub extensible arm64: kvm: Move lr save/restore from do_el2_call into EL1 arm64: kvm: deal with kernel symbols outside of linear mapping arm64: introduce KIMAGE_VADDR as the virtual base of the kernel region ANDROID: video: adf: Avoid directly referencing user pointers ANDROID: usb: gadget: audio_source: fix comparison of distinct pointer types android: binder: support for file-descriptor arrays. android: binder: support for scatter-gather. android: binder: add extra size to allocator. android: binder: refactor binder_transact() android: binder: support multiple /dev instances. android: binder: deal with contexts in debugfs. android: binder: support multiple context managers. android: binder: split flat_binder_object. disable aio support in recommended configuration Linux 4.4.32 scsi: megaraid_sas: fix macro MEGASAS_IS_LOGICAL to avoid regression drm/radeon: fix DP mode validation drm/radeon/dp: add back special handling for NUTMEG drm/amdgpu: fix DP mode validation drm/amdgpu/dp: add back special handling for NUTMEG KVM: MIPS: Drop other CPU ASIDs on guest MMU changes Revert KVM: MIPS: Drop other CPU ASIDs on guest MMU changes of: silence warnings due to max() usage packet: on direct_xmit, limit tso and csum to supported devices sctp: validate chunk len before actually using it net sched filters: fix notification of filter delete with proper handle udp: fix IP_CHECKSUM handling net: sctp, forbid negative length ipv4: use the right lock for ping_group_range ipv4: disable BH in set_ping_group_range() net: add recursion limit to GRO rtnetlink: Add rtnexthop offload flag to compare mask bridge: multicast: restore perm router ports on multicast enable net: pktgen: remove rcu locking in pktgen_change_name() ipv6: correctly add local routes when lo goes up ip6_tunnel: fix ip6_tnl_lookup ipv6: tcp: restore IP6CB for pktoptions skbs netlink: do not enter direct reclaim from netlink_dump() packet: call fanout_release, while UNREGISTERING a netdev net: Add netdev all_adj_list refcnt propagation to fix panic net/sched: act_vlan: Push skb->data to mac_header prior calling skb_vlan_*() functions net: pktgen: fix pkt_size net: fec: set mac address unconditionally tg3: Avoid NULL pointer dereference in tg3_io_error_detected() ipmr, ip6mr: fix scheduling while atomic and a deadlock with ipmr_get_route ip6_gre: fix flowi6_proto value in ip6gre_xmit_other() tcp: fix a compile error in DBGUNDO() tcp: fix wrong checksum calculation on MTU probing net: avoid sk_forward_alloc overflows tcp: fix overflow in __tcp_retransmit_skb() arm64/kvm: fix build issue on kvm debug arm64: ptdump: Indicate whether memory should be faulting arm64: Add support for ARCH_SUPPORTS_DEBUG_PAGEALLOC arm64: Drop alloc function from create_mapping arm64: allow vmalloc regions to be set with set_memory_* arm64: kernel: implement ACPI parking protocol arm64: mm: create new fine-grained mappings at boot arm64: ensure _stext and _etext are page-aligned arm64: mm: allow passing a pgdir to alloc_init_* arm64: mm: allocate pagetables anywhere arm64: mm: use fixmap when creating page tables arm64: mm: add functions to walk tables in fixmap arm64: mm: add __{pud,pgd}_populate arm64: mm: avoid redundant __pa(__va(x)) Linux 4.4.31 HID: usbhid: add ATEN CS962 to list of quirky devices ubi: fastmap: Fix add_vol() return value test in ubi_attach_fastmap() kvm: x86: Check memopp before dereference (CVE-2016-8630) tty: vt, fix bogus division in csi_J usb: dwc3: Fix size used in dma_free_coherent() pwm: Unexport children before chip removal UBI: fastmap: scrub PEB when bitflips are detected in a free PEB EC header Disable "frame-address" warning smc91x: avoid self-comparison warning cgroup: avoid false positive gcc-6 warning drm/exynos: fix error handling in exynos_drm_subdrv_open mm/cma: silence warnings due to max() usage ARM: 8584/1: floppy: avoid gcc-6 warning powerpc/ptrace: Fix out of bounds array access warning x86/xen: fix upper bound of pmd loop in xen_cleanhighmap() perf build: Fix traceevent plugins build race drm/dp/mst: Check peer device type before attempting EDID read drm/radeon: drop register readback in cayman_cp_int_cntl_setup drm/radeon/si_dpm: workaround for SI kickers drm/radeon/si_dpm: Limit clocks on HD86xx part Revert "drm/radeon: fix DP link training issue with second 4K monitor" mmc: dw_mmc-pltfm: fix the potential NULL pointer dereference scsi: arcmsr: Send SYNCHRONIZE_CACHE command to firmware scsi: scsi_debug: Fix memory leak if LBP enabled and module is unloaded scsi: megaraid_sas: Fix data integrity failure for JBOD (passthrough) devices mac80211: discard multicast and 4-addr A-MSDUs firewire: net: fix fragmented datagram_size off-by-one firewire: net: guard against rx buffer overflows Input: i8042 - add XMG C504 to keyboard reset table dm mirror: fix read error on recovery after default leg failure virtio: console: Unlock vqs while freeing buffers virtio_ring: Make interrupt suppression spec compliant parisc: Ensure consistent state when switching to kernel stack at syscall entry ovl: fsync after copy-up KVM: MIPS: Make ERET handle ERL before EXL KVM: x86: fix wbinvd_dirty_mask use-after-free dm: free io_barrier after blk_cleanup_queue call USB: serial: cp210x: fix tiocmget error handling tty: limit terminal size to 4M chars xhci: add restart quirk for Intel Wildcatpoint PCH hv: do not lose pending heartbeat vmbus packets vt: clear selection before resizing Fix potential infoleak in older kernels GenWQE: Fix bad page access during abort of resource allocation usb: increase ohci watchdog delay to 275 msec xhci: use default USB_RESUME_TIMEOUT when resuming ports. USB: serial: ftdi_sio: add support for Infineon TriBoard TC2X7 USB: serial: fix potential NULL-dereference at probe usb: gadget: function: u_ether: don't starve tx request queue mei: txe: don't clean an unprocessed interrupt cause. ubifs: Fix regression in ubifs_readdir() ubifs: Abort readdir upon error btrfs: fix races on root_log_ctx lists ANDROID: binder: Clear binder and cookie when setting handle in flat binder struct ANDROID: binder: Add strong ref checks ALSA: hda - Fix headset mic detection problem for two Dell laptops ALSA: hda - Adding a new group of pin cfg into ALC295 pin quirk table ALSA: hda - allow 40 bit DMA mask for NVidia devices ALSA: hda - Raise AZX_DCAPS_RIRB_DELAY handling into top drivers ALSA: hda - Merge RIRB_PRE_DELAY into CTX_WORKAROUND caps ALSA: usb-audio: Add quirk for Syntek STK1160 KEYS: Fix short sprintf buffer in /proc/keys show function mm: memcontrol: do not recurse in direct reclaim mm/list_lru.c: avoid error-path NULL pointer deref libxfs: clean up _calc_dquots_per_chunk h8300: fix syscall restarting drm/dp/mst: Clear port->pdt when tearing down the i2c adapter i2c: core: fix NULL pointer dereference under race condition i2c: xgene: Avoid dma_buffer overrun arm64:cpufeature ARM64_NCAPS is the indicator of last feature arm64: hibernate: Refuse to hibernate if the boot cpu is offline PM / sleep: Add support for read-only sysfs attributes arm64: kernel: Add support for hibernate/suspend-to-disk arm64: mm: add functions to walk page tables by PA arm64: mm: move pte_* macros PM / Hibernate: Call flush_icache_range() on pages restored in-place arm64: Add new asm macro copy_page arm64: Promote KERNEL_START/KERNEL_END definitions to a header file arm64: kernel: Include _AC definition in page.h arm64: Change cpu_resume() to enable mmu early then access sleep_sp by va arm64: kernel: Rework finisher callback out of __cpu_suspend_enter() arm64: Cleanup SCTLR flags arm64: Fold proc-macros.S into assembler.h arm/arm64: KVM: Add hook for C-based stage2 init arm/arm64: KVM: Detect vGIC presence at runtime arm64: KVM: Add support for 16-bit VMID arm: KVM: Make kvm_arm.h friendly to assembly code arm/arm64: KVM: Remove unreferenced S2_PGD_ORDER arm64: KVM: debug: Remove spurious inline attributes ARM: KVM: Cleanup exception injection arm64: KVM: Remove weak attributes arm64: KVM: Cleanup asm-offset.c arm64: KVM: Turn system register numbers to an enum arm64: KVM: VHE: Patch out use of HVC arm64: Add ARM64_HAS_VIRT_HOST_EXTN feature arm/arm64: Add new is_kernel_in_hyp_mode predicate arm64: KVM: Move away from the assembly version of the world switch arm64: KVM: Map the kernel RO section into HYP arm64: KVM: Add compatibility aliases arm64: KVM: Implement vgic-v3 save/restore arm64: KVM: Add panic handling arm64: KVM: HYP mode entry points arm64: KVM: Implement TLB handling arm64: KVM: Implement fpsimd save/restore arm64: KVM: Implement the core world switch arm64: KVM: Add patchable function selector arm64: KVM: Implement guest entry arm64: KVM: Implement debug save/restore arm64: KVM: Implement 32bit system register save/restore arm64: KVM: Implement system register save/restore arm64: KVM: Implement timer save/restore arm64: KVM: Implement vgic-v2 save/restore arm64: KVM: Add a HYP-specific header file KVM: arm/arm64: vgic-v3: Make the LR indexing macro public arm64: Add macros to read/write system registers Linux 4.4.30 Revert "fix minor infoleak in get_user_ex()" Revert "x86/mm: Expand the exception table logic to allow new handling options" Linux 4.4.29 ARM: pxa: pxa_cplds: fix interrupt handling powerpc/nvram: Fix an incorrect partition merge mpt3sas: Don't spam logs if logging level is 0 perf symbols: Fixup symbol sizes before picking best ones perf symbols: Check symbol_conf.allow_aliases for kallsyms loading too perf hists browser: Fix event group display clk: divider: Fix clk_divider_round_rate() to use clk_readl() clk: qoriq: fix a register offset error s390/con3270: fix insufficient space padding s390/con3270: fix use of uninitialised data s390/cio: fix accidental interrupt enabling during resume x86/mm: Expand the exception table logic to allow new handling options dmaengine: ipu: remove bogus NO_IRQ reference power: bq24257: Fix use of uninitialized pointer bq->charger staging: r8188eu: Fix scheduling while atomic splat ASoC: dapm: Fix kcontrol creation for output driver widget ASoC: dapm: Fix value setting for _ENUM_DOUBLE MUX's second channel ASoC: dapm: Fix possible uninitialized variable in snd_soc_dapm_get_volsw() ASoC: topology: Fix error return code in soc_tplg_dapm_widget_create() hwrng: omap - Only fail if pm_runtime_get_sync returns < 0 crypto: arm/ghash-ce - add missing async import/export crypto: gcm - Fix IV buffer size in crypto_gcm_setkey mwifiex: correct aid value during tdls setup spi: spi-fsl-dspi: Drop extra spi_master_put in device remove function ARM: clk-imx35: fix name for ckil clk uio: fix dmem_region_start computation genirq/generic_chip: Add irq_unmap callback perf stat: Fix interval output values powerpc/eeh: Null check uses of eeh_pe_bus_get tunnels: Remove encapsulation offloads on decap. tunnels: Don't apply GRO to multiple layers of encapsulation. ipip: Properly mark ipip GRO packets as encapsulated. posix_acl: Clear SGID bit when setting file permissions brcmfmac: avoid potential stack overflow in brcmf_cfg80211_start_ap() mm/hugetlb: fix memory offline with hugepage size > memory block size drm/i915: Unalias obj->phys_handle and obj->userptr drm/i915: Account for TSEG size when determining 865G stolen base Revert "drm/i915: Check live status before reading edid" drm/i915/gen9: fix the WaWmMemoryReadLatency implementation xenbus: don't look up transaction IDs for ordinary writes drm/vmwgfx: Limit the user-space command buffer size drm/radeon: change vblank_time's calculation method to reduce computational error. drm/radeon/si/dpm: fix phase shedding setup drm/radeon: narrow asic_init for virtualization drm/amdgpu: change vblank_time's calculation method to reduce computational error. drm/amdgpu/dce11: add missing drm_mode_config_cleanup call drm/amdgpu/dce11: disable hpd on local panels drm/amdgpu/dce8: disable hpd on local panels drm/amdgpu/dce10: disable hpd on local panels drm/amdgpu: fix IB alignment for UVD drm/prime: Pass the right module owner through to dma_buf_export() Linux 4.4.28 target: Don't override EXTENDED_COPY xcopy_pt_cmd SCSI status code target: Make EXTENDED_COPY 0xe4 failure return COPY TARGET DEVICE NOT REACHABLE target: Re-add missing SCF_ACK_KREF assignment in v4.1.y ubifs: Fix xattr_names length in exit paths jbd2: fix incorrect unlock on j_list_lock ext4: do not advertise encryption support when disabled mmc: rtsx_usb_sdmmc: Handle runtime PM while changing the led mmc: rtsx_usb_sdmmc: Avoid keeping the device runtime resumed when unused mmc: core: Annotate cmd_hdr as __le32 powerpc/mm: Prevent unlikely crash in copro_calculate_slb() ceph: fix error handling in ceph_read_iter arm64: kernel: Init MDCR_EL2 even in the absence of a PMU arm64: percpu: rewrite ll/sc loops in assembly memstick: rtsx_usb_ms: Manage runtime PM when accessing the device memstick: rtsx_usb_ms: Runtime resume the device when polling for cards isofs: Do not return EACCES for unknown filesystems irqchip/gic-v3-its: Fix entry size mask for GITS_BASER s390/mm: fix gmap tlb flush issues Using BUG_ON() as an assert() is _never_ acceptable mm: filemap: fix mapping->nrpages double accounting in fuse mm: workingset: fix crash in shadow node shrinker caused by replace_page_cache_page() acpi, nfit: check for the correct event code in notifications net/mlx4_core: Allow resetting VF admin mac to zero bnx2x: Prevent false warning for lack of FC NPIV PKCS#7: Don't require SpcSpOpusInfo in Authenticode pkcs7 signatures hpsa: correct skipping masked peripherals sd: Fix rw_max for devices that report an optimal xfer size irqchip/gicv3: Handle loop timeout proper kvm: x86: memset whole irq_eoi x86/e820: Don't merge consecutive E820_PRAM ranges blkcg: Unlock blkcg_pol_mutex only once when cpd == NULL Fix regression which breaks DFS mounting Cleanup missing frees on some ioctls Do not send SMB3 SET_INFO request if nothing is changing SMB3: GUIDs should be constructed as random but valid uuids Set previous session id correctly on SMB3 reconnect Display number of credits available Clarify locking of cifs file and tcon structures and make more granular fs/cifs: keep guid when assigning fid to fileinfo cifs: Limit the overall credit acquired fs/super.c: fix race between freeze_super() and thaw_super() arc: don't leak bits of kernel stack into coredump lightnvm: ensure that nvm_dev_ops can be used without CONFIG_NVM ipc/sem.c: fix complex_count vs. simple op race mm: filemap: don't plant shadow entries without radix tree node metag: Only define atomic_dec_if_positive conditionally scsi: Fix use-after-free NFSv4.2: Fix a reference leak in nfs42_proc_layoutstats_generic NFSv4: Open state recovery must account for file permission changes NFSv4: nfs4_copy_delegation_stateid() must fail if the delegation is invalid NFSv4: Don't report revoked delegations as valid in nfs_have_delegation() sunrpc: fix write space race causing stalls Input: elantech - add Fujitsu Lifebook E556 to force crc_enabled Input: elantech - force needed quirks on Fujitsu H760 Input: i8042 - skip selftest on ASUS laptops lib: add "on"/"off" support to kstrtobool lib: update single-char callers of strtobool() lib: move strtobool() to kstrtobool() MIPS: ptrace: Fix regs_return_value for kernel context MIPS: Fix -mabi=64 build of vdso.lds ALSA: hda - Fix a failure of micmute led when having multi adcs cx231xx: fix GPIOs for Pixelview SBTVD hybrid cx231xx: don't return error on success mb86a20s: fix demod settings mb86a20s: fix the locking logic ovl: copy_up_xattr(): use strnlen ovl: Fix info leak in ovl_lookup_temp() fbdev/efifb: Fix 16 color palette entry calculation scsi: zfcp: spin_lock_irqsave() is not nestable zfcp: trace full payload of all SAN records (req,resp,iels) zfcp: fix payload trace length for SAN request&response zfcp: fix D_ID field with actual value on tracing SAN responses zfcp: restore tracing of handle for port and LUN with HBA records zfcp: trace on request for open and close of WKA port zfcp: restore: Dont use 0 to indicate invalid LUN in rec trace zfcp: retain trace level for SCSI and HBA FSF response records zfcp: close window with unblocked rport during rport gone zfcp: fix ELS/GS request&response length for hardware data router zfcp: fix fc_host port_type with NPIV ubi: Deal with interrupted erasures in WL powerpc/pseries: Fix stack corruption in htpe code powerpc/64: Fix incorrect return value from __copy_tofrom_user powerpc/powernv: Use CPU-endian PEST in pnv_pci_dump_p7ioc_diag_data() powerpc/powernv: Use CPU-endian hub diag-data type in pnv_eeh_get_and_dump_hub_diag() powerpc/powernv: Pass CPU-endian PE number to opal_pci_eeh_freeze_clear() powerpc/vdso64: Use double word compare on pointers dm crypt: fix crash on exit dm mpath: check if path's request_queue is dying in activate_path() dm: return correct error code in dm_resume()'s retry loop dm: mark request_queue dead before destroying the DM device perf intel-pt: Fix MTC timestamp calculation for large MTC periods perf intel-pt: Fix estimated timestamps for cycle-accurate mode perf intel-pt: Fix snapshot overlap detection decoder errors pstore/ram: Use memcpy_fromio() to save old buffer pstore/ram: Use memcpy_toio instead of memcpy pstore/core: drop cmpxchg based updates pstore/ramoops: fixup driver removal parisc: Increase initial kernel mapping size parisc: Fix kernel memory layout regarding position of __gp parisc: Increase KERNEL_INITIAL_SIZE for 32-bit SMP kernels cpufreq: intel_pstate: Fix unsafe HWP MSR access platform: don't return 0 from platform_get_irq[_byname]() on error PCI: Mark Atheros AR9580 to avoid bus reset mmc: sdhci: cast unsigned int to unsigned long long to avoid unexpeted error mmc: block: don't use CMD23 with very old MMC cards rtlwifi: Fix missing country code for Great Britain PM / devfreq: event: remove duplicate devfreq_event_get_drvdata() clk: imx6: initialize GPU clocks regulator: tps65910: Work around silicon erratum SWCZ010 mei: me: add kaby point device ids gpio: mpc8xxx: Correct irq handler function cgroup: Change from CAP_SYS_NICE to CAP_SYS_RESOURCE for cgroup migration permissions UPSTREAM: cpu/hotplug: Handle unbalanced hotplug enable/disable UPSTREAM: arm64: kaslr: fix breakage with CONFIG_MODVERSIONS=y UPSTREAM: arm64: kaslr: keep modules close to the kernel when DYNAMIC_FTRACE=y cgroup: Remove leftover instances of allow_attach BACKPORT: lib: harden strncpy_from_user CHROMIUM: cgroups: relax permissions on moving tasks between cgroups CHROMIUM: remove Android's cgroup generic permissions checks Linux 4.4.27 cfq: fix starvation of asynchronous writes vfs: move permission checking into notify_change() for utimes(NULL) dlm: free workqueues after the connections crypto: vmx - Fix memory corruption caused by p8_ghash crypto: ghash-generic - move common definitions to a new header file ext4: release bh in make_indexed_dir ext4: allow DAX writeback for hole punch ext4: fix memory leak in ext4_insert_range() ext4: reinforce check of i_dtime when clearing high fields of uid and gid ext4: enforce online defrag restriction for encrypted files scsi: ibmvfc: Fix I/O hang when port is not mapped scsi: arcmsr: Simplify user_len checking scsi: arcmsr: Buffer overflow in arcmsr_iop_message_xfer() async_pq_val: fix DMA memory leak reiserfs: switch to generic_{get,set,remove}xattr() reiserfs: Unlock superblock before calling reiserfs_quota_on_mount() ASoC: Intel: Atom: add a missing star in a memcpy call brcmfmac: fix memory leak in brcmf_fill_bss_param i40e: avoid NULL pointer dereference and recursive errors on early PCI error fuse: fix killing s[ug]id in setattr fuse: invalidate dir dentry after chmod fuse: listxattr: verify xattr list drivers: base: dma-mapping: page align the size when unmap_kernel_range btrfs: assign error values to the correct bio structs serial: 8250_dw: Check the data->pclk when get apb_pclk arm64: Use PoU cache instr for I/D coherency arm64: mm: add code to safely replace TTBR1_EL1 arm64: mm: place __cpu_setup in .text arm64: add function to install the idmap arm64: unmap idmap earlier arm64: unify idmap removal arm64: mm: place empty_zero_page in bss arm64: head.S: use memset to clear BSS arm64: mm: specialise pagetable allocators arm64: mm: remove pointless PAGE_MASKing asm-generic: Fix local variable shadow in __set_fixmap_offset arm64: mm: fold alternatives into .init ARM: 8511/1: ARM64: kernel: PSCI: move PSCI idle management code to drivers/firmware ARM: 8481/2: drivers: psci: replace psci firmware calls ARM: 8480/2: arm64: add implementation for arm-smccc ARM: 8479/2: add implementation for arm-smccc ARM: 8478/2: arm/arm64: add arm-smccc ARM: 8510/1: rework ARM_CPU_SUSPEND dependencies ARM: 8458/1: bL_switcher: add GIC dependency Linux 4.4.26 mm: remove gup_flags FOLL_WRITE games from __get_user_pages() x86/build: Build compressed x86 kernels as PIE arm64: Remove stack duplicating code from jprobes arm64: kprobes: Add KASAN instrumentation around stack accesses arm64: kprobes: Cleanup jprobe_return arm64: kprobes: Fix overflow when saving stack arm64: kprobes: WARN if attempting to step with PSTATE.D=1 kprobes: Add arm64 case in kprobe example module arm64: Add kernel return probes support (kretprobes) arm64: Add trampoline code for kretprobes arm64: kprobes instruction simulation support arm64: Treat all entry code as non-kprobe-able arm64: Blacklist non-kprobe-able symbol arm64: Kprobes with single stepping support arm64: add conditional instruction simulation support arm64: Add more test functions to insn.c arm64: Add HAVE_REGS_AND_STACK_ACCESS_API feature Linux 4.4.25 tpm_crb: fix crb_req_canceled behavior tpm: fix a race condition in tpm2_unseal_trusted() ima: use file_dentry() ARM: cpuidle: Fix error return code ARM: dts: MSM8064 remove flags from SPMI/MPP IRQs ARM: dts: mvebu: armada-390: add missing compatibility string and bracket x86/dumpstack: Fix x86_32 kernel_stack_pointer() previous stack access x86/irq: Prevent force migration of irqs which are not in the vector domain x86/boot: Fix kdump, cleanup aborted E820_PRAM max_pfn manipulation KVM: PPC: BookE: Fix a sanity check KVM: MIPS: Drop other CPU ASIDs on guest MMU changes KVM: PPC: Book3s PR: Allow access to unprivileged MMCR2 register mfd: wm8350-i2c: Make sure the i2c regmap functions are compiled mfd: 88pm80x: Double shifting bug in suspend/resume mfd: atmel-hlcdc: Do not sleep in atomic context mfd: rtsx_usb: Avoid setting ucr->current_sg.status ALSA: usb-line6: use the same declaration as definition in header for MIDI manufacturer ID ALSA: usb-audio: Extend DragonFly dB scale quirk to cover other variants ALSA: ali5451: Fix out-of-bound position reporting timekeeping: Fix __ktime_get_fast_ns() regression time: Add cycles to nanoseconds translation mm: Fix build for hardened usercopy ANDROID: binder: Clear binder and cookie when setting handle in flat binder struct ANDROID: binder: Add strong ref checks UPSTREAM: staging/android/ion : fix a race condition in the ion driver ANDROID: android-base: CONFIG_HARDENED_USERCOPY=y UPSTREAM: fs/proc/kcore.c: Add bounce buffer for ktext data UPSTREAM: fs/proc/kcore.c: Make bounce buffer global for read BACKPORT: arm64: Correctly bounds check virt_addr_valid Fix a build breakage in IO latency hist code. UPSTREAM: efi: include asm/early_ioremap.h not asm/efi.h to get early_memremap UPSTREAM: ia64: split off early_ioremap() declarations into asm/early_ioremap.h FROMLIST: arm64: Enable CONFIG_ARM64_SW_TTBR0_PAN FROMLIST: arm64: xen: Enable user access before a privcmd hvc call FROMLIST: arm64: Handle faults caused by inadvertent user access with PAN enabled FROMLIST: arm64: Disable TTBR0_EL1 during normal kernel execution FROMLIST: arm64: Introduce uaccess_{disable,enable} functionality based on TTBR0_EL1 FROMLIST: arm64: Factor out TTBR0_EL1 post-update workaround into a specific asm macro FROMLIST: arm64: Factor out PAN enabling/disabling into separate uaccess_* macros UPSTREAM: arm64: Handle el1 synchronous instruction aborts cleanly UPSTREAM: arm64: include alternative handling in dcache_by_line_op UPSTREAM: arm64: fix "dc cvau" cache operation on errata-affected core UPSTREAM: Revert "arm64: alternatives: add enable parameter to conditional asm macros" UPSTREAM: arm64: Add new asm macro copy_page UPSTREAM: arm64: kill ESR_LNX_EXEC UPSTREAM: arm64: add macro to extract ESR_ELx.EC UPSTREAM: arm64: mm: mark fault_info table const UPSTREAM: arm64: fix dump_instr when PAN and UAO are in use BACKPORT: arm64: Fold proc-macros.S into assembler.h UPSTREAM: arm64: choose memstart_addr based on minimum sparsemem section alignment UPSTREAM: arm64/mm: ensure memstart_addr remains sufficiently aligned UPSTREAM: arm64/kernel: fix incorrect EL0 check in inv_entry macro UPSTREAM: arm64: Add macros to read/write system registers UPSTREAM: arm64/efi: refactor EFI init and runtime code for reuse by 32-bit ARM UPSTREAM: arm64/efi: split off EFI init and runtime code for reuse by 32-bit ARM UPSTREAM: arm64/efi: mark UEFI reserved regions as MEMBLOCK_NOMAP BACKPORT: arm64: only consider memblocks with NOMAP cleared for linear mapping UPSTREAM: mm/memblock: add MEMBLOCK_NOMAP attribute to memblock memory table ANDROID: dm: android-verity: Remove fec_header location constraint BACKPORT: audit: consistently record PIDs with task_tgid_nr() android-base.cfg: Enable kernel ASLR UPSTREAM: vmlinux.lds.h: allow arch specific handling of ro_after_init data section UPSTREAM: arm64: spinlock: fix spin_unlock_wait for LSE atomics UPSTREAM: arm64: avoid TLB conflict with CONFIG_RANDOMIZE_BASE UPSTREAM: arm64: Only select ARM64_MODULE_PLTS if MODULES=y sched: Add Kconfig option DEFAULT_USE_ENERGY_AWARE to set ENERGY_AWARE feature flag sched/fair: remove printk while schedule is in progress ANDROID: fs: FS tracepoints to track IO. sched/walt: Drop arch-specific timer access ANDROID: fiq_debugger: Pass task parameter to unwind_frame() eas/sched/fair: Fixing comments in find_best_target. input: keyreset: switch to orderly_reboot UPSTREAM: tun: fix transmit timestamp support UPSTREAM: arch/arm/include/asm/pgtable-3level.h: add pmd_mkclean for THP net: inet: diag: expose the socket mark to privileged processes. net: diag: make udp_diag_destroy work for mapped addresses. net: diag: support SOCK_DESTROY for UDP sockets net: diag: allow socket bytecode filters to match socket marks net: diag: slightly refactor the inet_diag_bc_audit error checks. net: diag: Add support to filter on device index UPSTREAM: brcmfmac: avoid potential stack overflow in brcmf_cfg80211_start_ap() Linux 4.4.24 ALSA: hda - Add the top speaker pin config for HP Spectre x360 ALSA: hda - Fix headset mic detection problem for several Dell laptops ACPICA: acpi_get_sleep_type_data: Reduce warnings ALSA: hda - Adding one more ALC255 pin definition for headset problem Revert "usbtmc: convert to devm_kzalloc" USB: serial: cp210x: Add ID for a Juniper console Staging: fbtft: Fix bug in fbtft-core usb: misc: legousbtower: Fix NULL pointer deference USB: serial: cp210x: fix hardware flow-control disable dm log writes: fix bug with too large bios clk: xgene: Add missing parenthesis when clearing divider value aio: mark AIO pseudo-fs noexec batman-adv: remove unused callback from batadv_algo_ops struct IB/mlx4: Use correct subnet-prefix in QP1 mads under SR-IOV IB/mlx4: Fix code indentation in QP1 MAD flow IB/mlx4: Fix incorrect MC join state bit-masking on SR-IOV IB/ipoib: Don't allow MC joins during light MC flush IB/core: Fix use after free in send_leave function IB/ipoib: Fix memory corruption in ipoib cm mode connect flow KVM: nVMX: postpone VMCS changes on MSR_IA32_APICBASE write dmaengine: at_xdmac: fix to pass correct device identity to free_irq() kernel/fork: fix CLONE_CHILD_CLEARTID regression in nscd ASoC: omap-mcpdm: Fix irq resource handling sysctl: handle error writing UINT_MAX to u32 fields powerpc/prom: Fix sub-processor option passed to ibm, client-architecture-support brcmsmac: Initialize power in brcms_c_stf_ss_algo_channel_get() brcmsmac: Free packet if dma_mapping_error() fails in dma_rxfill brcmfmac: Fix glob_skb leak in brcmf_sdiod_recv_chain ASoC: Intel: Skylake: Fix error return code in skl_probe() pNFS/flexfiles: Fix layoutcommit after a commit to DS pNFS/files: Fix layoutcommit after a commit to DS NFS: Don't drop CB requests with invalid principals svc: Avoid garbage replies when pc_func() returns rpc_drop_reply dmaengine: at_xdmac: fix debug string fnic: pci_dma_mapping_error() doesn't return an error code avr32: off by one in at32_init_pio() ath9k: Fix programming of minCCA power threshold gspca: avoid unused variable warnings em28xx-i2c: rt_mutex_trylock() returns zero on failure NFC: fdp: Detect errors from fdp_nci_create_conn() iwlmvm: mvm: set correct state in smart-fifo configuration tile: Define AT_VECTOR_SIZE_ARCH for ARCH_DLINFO pstore: drop file opened reference count blk-mq: actually hook up defer list when running requests hwrng: omap - Fix assumption that runtime_get_sync will always succeed ARM: sa1111: fix pcmcia suspend/resume ARM: shmobile: fix regulator quirk for Gen2 ARM: sa1100: clear reset status prior to reboot ARM: sa1100: fix 3.6864MHz clock ARM: sa1100: register clocks early ARM: sun5i: Fix typo in trip point temperature regulator: qcom_smd: Fix voltage ranges for pm8x41 regulator: qcom_spmi: Update mvs1/mvs2 switches on pm8941 regulator: qcom_spmi: Add support for get_mode/set_mode on switches regulator: qcom_spmi: Add support for S4 supply on pm8941 tpm: fix byte-order for the value read by tpm2_get_tpm_pt printk: fix parsing of "brl=" option MIPS: uprobes: fix use of uninitialised variable MIPS: Malta: Fix IOCU disable switch read for MIPS64 MIPS: fix uretprobe implementation MIPS: uprobes: remove incorrect set_orig_insn arm64: debug: avoid resetting stepping state machine when TIF_SINGLESTEP ARM: 8618/1: decompressor: reset ttbcr fields to use TTBR0 on ARMv7 irqchip/gicv3: Silence noisy DEBUG_PER_CPU_MAPS warning gpio: sa1100: fix irq probing for ucb1x00 usb: gadget: fsl_qe_udc: signedness bug in qe_get_frame() ceph: fix race during filling readdir cache iwlwifi: mvm: don't use ret when not initialised iwlwifi: pcie: fix access to scratch buffer spi: sh-msiof: Avoid invalid clock generator parameters hwmon: (adt7411) set bit 3 in CFG1 register nvmem: Declare nvmem_cell_read() consistently ipvs: fix bind to link-local mcast IPv6 address in backup tools/vm/slabinfo: fix an unintentional printf mmc: pxamci: fix potential oops drivers/perf: arm_pmu: Fix leak in error path pinctrl: Flag strict is a field in struct pinmux_ops pinctrl: uniphier: fix .pin_dbg_show() callback i40e: avoid null pointer dereference perf/core: Fix pmu::filter_match for SW-led groups iwlwifi: mvm: fix a few firmware capability checks usb: musb: fix DMA for host mode usb: musb: Fix DMA desired mode for Mentor DMA engine ARM: 8617/1: dma: fix dma_max_pfn() ARM: 8616/1: dt: Respect property size when parsing CPUs drm/radeon/si/dpm: add workaround for for Jet parts drm/nouveau/fifo/nv04: avoid ramht race against cookie insertion x86/boot: Initialize FPU and X86_FEATURE_ALWAYS even if we don't have CPUID x86/init: Fix cr4_init_shadow() on CR4-less machines can: dev: fix deadlock reported after bus-off mm,ksm: fix endless looping in allocating memory when ksm enable mtd: nand: davinci: Reinitialize the HW ECC engine in 4bit hwctl cpuset: handle race between CPU hotplug and cpuset_hotplug_work usercopy: fold builtin_const check into inline function Linux 4.4.23 hostfs: Freeing an ERR_PTR in hostfs_fill_sb_common() qxl: check for kmap failures power: supply: max17042_battery: fix model download bug. power_supply: tps65217-charger: fix missing platform_set_drvdata() PM / hibernate: Fix rtree_next_node() to avoid walking off list ends PM / hibernate: Restore processor state before using per-CPU variables MIPS: paravirt: Fix undefined reference to smp_bootstrap MIPS: Add a missing ".set pop" in an early commit MIPS: Avoid a BUG warning during prctl(PR_SET_FP_MODE, ...) MIPS: Remove compact branch policy Kconfig entries MIPS: vDSO: Fix Malta EVA mapping to vDSO page structs MIPS: SMP: Fix possibility of deadlock when bringing CPUs online MIPS: Fix pre-r6 emulation FPU initialisation i2c: qup: skip qup_i2c_suspend if the device is already runtime suspended i2c-eg20t: fix race between i2c init and interrupt enable btrfs: ensure that file descriptor used with subvol ioctls is a dir nl80211: validate number of probe response CSA counters can: flexcan: fix resume function mm: delete unnecessary and unsafe init_tlb_ubc() tracing: Move mutex to protect against resetting of seq data fix memory leaks in tracing_buffers_splice_read() power: reset: hisi-reboot: Unmap region obtained by of_iomap mtd: pmcmsp-flash: Allocating too much in init_msp_flash() mtd: maps: sa1100-flash: potential NULL dereference fix fault_in_multipages_...() on architectures with no-op access_ok() fanotify: fix list corruption in fanotify_get_response() fsnotify: add a way to stop queueing events on group shutdown xfs: prevent dropping ioend completions during buftarg wait autofs: use dentry flags to block walks during expire autofs races pwm: Mark all devices as "might sleep" bridge: re-introduce 'fix parsing of MLDv2 reports' net: smc91x: fix SMC accesses Revert "phy: IRQ cannot be shared" net: dsa: bcm_sf2: Fix race condition while unmasking interrupts net/mlx5: Added missing check of msg length in verifying its signature tipc: fix NULL pointer dereference in shutdown() net/irda: handle iriap_register_lsap() allocation failure vti: flush x-netns xfrm cache when vti interface is removed af_unix: split 'u->readlock' into two: 'iolock' and 'bindlock' Revert "af_unix: Fix splice-bind deadlock" bonding: Fix bonding crash megaraid: fix null pointer check in megasas_detach_one(). nouveau: fix nv40_perfctr_next() cleanup regression Staging: iio: adc: fix indent on break statement iwlegacy: avoid warning about missing braces ath9k: fix misleading indentation am437x-vfpe: fix typo in vpfe_get_app_input_index Add braces to avoid "ambiguous ‘else’" compiler warnings net: caif: fix misleading indentation Makefile: Mute warning for __builtin_return_address(>0) for tracing only Disable "frame-address" warning Disable "maybe-uninitialized" warning globally gcov: disable -Wmaybe-uninitialized warning Kbuild: disable 'maybe-uninitialized' warning for CONFIG_PROFILE_ALL_BRANCHES kbuild: forbid kernel directory to contain spaces and colons tools: Support relative directory path for 'O=' Makefile: revert "Makefile: Document ability to make file.lst and file.S" partially kbuild: Do not run modules_install and install in paralel ocfs2: fix start offset to ocfs2_zero_range_for_truncate() ocfs2/dlm: fix race between convert and migration crypto: echainiv - Replace chaining with multiplication crypto: skcipher - Fix blkcipher walk OOM crash crypto: arm/aes-ctr - fix NULL dereference in tail processing crypto: arm64/aes-ctr - fix NULL dereference in tail processing tcp: properly scale window in tcp_v[46]_reqsk_send_ack() tcp: fix use after free in tcp_xmit_retransmit_queue() tcp: cwnd does not increase in TCP YeAH ipv6: release dst in ping_v6_sendmsg ipv4: panic in leaf_walk_rcu due to stale node pointer reiserfs: fix "new_insert_key may be used uninitialized ..." Fix build warning in kernel/cpuset.c include/linux/kernel.h: change abs() macro so it uses consistent return type Linux 4.4.22 openrisc: fix the fix of copy_from_user() avr32: fix 'undefined reference to `___copy_from_user' ia64: copy_from_user() should zero the destination on access_ok() failure genirq/msi: Fix broken debug output ppc32: fix copy_from_user() sparc32: fix copy_from_user() mn10300: copy_from_user() should zero on access_ok() failure... nios2: copy_from_user() should zero the tail of destination openrisc: fix copy_from_user() parisc: fix copy_from_user() metag: copy_from_user() should zero the destination on access_ok() failure alpha: fix copy_from_user() asm-generic: make copy_from_user() zero the destination properly mips: copy_from_user() must zero the destination on access_ok() failure hexagon: fix strncpy_from_user() error return sh: fix copy_from_user() score: fix copy_from_user() and friends blackfin: fix copy_from_user() cris: buggered copy_from_user/copy_to_user/clear_user frv: fix clear_user() asm-generic: make get_user() clear the destination on errors ARC: uaccess: get_user to zero out dest in cause of fault s390: get_user() should zero on failure score: fix __get_user/get_user nios2: fix __get_user() sh64: failing __get_user() should zero m32r: fix __get_user() mn10300: failing __get_user() and get_user() should zero fix minor infoleak in get_user_ex() microblaze: fix copy_from_user() avr32: fix copy_from_user() microblaze: fix __get_user() fix iov_iter_fault_in_readable() irqchip/atmel-aic: Fix potential deadlock in ->xlate() genirq: Provide irq_gc_{lock_irqsave,unlock_irqrestore}() helpers drm: Only use compat ioctl for addfb2 on X86/IA64 drm: atmel-hlcdc: Fix vertical scaling net: simplify napi_synchronize() to avoid warnings kconfig: tinyconfig: provide whole choice blocks to avoid warnings soc: qcom/spm: shut up uninitialized variable warning pinctrl: at91-pio4: use %pr format string for resource mmc: dw_mmc: use resource_size_t to store physical address drm/i915: Avoid pointer arithmetic in calculating plane surface offset mpssd: fix buffer overflow warning gma500: remove annoying deprecation warning ipv6: addrconf: fix dev refcont leak when DAD failed sched/core: Fix a race between try_to_wake_up() and a woken up task Revert "wext: Fix 32 bit iwpriv compatibility issue with 64 bit Kernel" ath9k: fix using sta->drv_priv before initializing it md-cluster: make md-cluster also can work when compiled into kernel xhci: fix null pointer dereference in stop command timeout function fuse: direct-io: don't dirty ITER_BVEC pages Btrfs: remove root_log_ctx from ctx list before btrfs_sync_log returns crypto: cryptd - initialize child shash_desc on import arm64: spinlocks: implement smp_mb__before_spinlock() as smp_mb() pinctrl: sunxi: fix uart1 CTS/RTS pins at PG on A23/A33 pinctrl: pistachio: fix mfio pll_lock pinmux dm crypt: fix error with too large bios dm log writes: move IO accounting earlier to fix error path dm log writes: fix check of kthread_run() return value bus: arm-ccn: Fix XP watchpoint settings bitmask bus: arm-ccn: Do not attempt to configure XPs for cycle counter bus: arm-ccn: Fix PMU handling of MN ARM: dts: STiH407-family: Provide interconnect clock for consumption in ST SDHCI ARM: dts: overo: fix gpmc nand on boards with ethernet ARM: dts: overo: fix gpmc nand cs0 range ARM: dts: imx6qdl: Fix SPDIF regression ARM: OMAP3: hwmod data: Add sysc information for DSI ARM: kirkwood: ib62x0: fix size of u-boot environment partition ARM: imx6: add missing BM_CLPCR_BYPASS_PMIC_READY setting for imx6sx ARM: imx6: add missing BM_CLPCR_BYP_MMDC_CH0_LPM_HS setting for imx6ul ARM: AM43XX: hwmod: Fix RSTST register offset for pruss cpuset: make sure new tasks conform to the current config of the cpuset net: thunderx: Fix OOPs with ethtool --register-dump USB: change bInterval default to 10 ms ARM: dts: STiH410: Handle interconnect clock required by EHCI/OHCI (USB) usb: chipidea: udc: fix NULL ptr dereference in isr_setup_status_phase usb: renesas_usbhs: fix clearing the {BRDY,BEMP}STS condition USB: serial: simple: add support for another Infineon flashloader serial: 8250: added acces i/o products quad and octal serial cards serial: 8250_mid: fix divide error bug if baud rate is 0 iio: ensure ret is initialized to zero before entering do loop iio:core: fix IIO_VAL_FRACTIONAL sign handling iio: accel: kxsd9: Fix scaling bug iio: fix pressure data output unit in hid-sensor-attributes iio: accel: bmc150: reset chip at init time iio: adc: at91: unbreak channel adc channel 3 iio: ad799x: Fix buffered capture for ad7991/ad7995/ad7999 iio: adc: ti_am335x_adc: Increase timeout value waiting for ADC sample iio: adc: ti_am335x_adc: Protect FIFO1 from concurrent access iio: adc: rockchip_saradc: reset saradc controller before programming it iio: proximity: as3935: set up buffer timestamps for non-zero values iio: accel: kxsd9: Fix raw read return kvm-arm: Unmap shadow pagetables properly x86/AMD: Apply erratum 665 on machines without a BIOS fix x86/paravirt: Do not trace _paravirt_ident_*() functions ARC: mm: fix build breakage with STRICT_MM_TYPECHECKS IB/uverbs: Fix race between uverbs_close and remove_one dm flakey: fix reads to be issued if drop_writes configured audit: fix exe_file access in audit_exe_compare mm: introduce get_task_exe_file kexec: fix double-free when failing to relocate the purgatory NFSv4.1: Fix the CREATE_SESSION slot number accounting pNFS: Ensure LAYOUTGET and LAYOUTRETURN are properly serialised nfsd: Close race between nfsd4_release_lockowner and nfsd4_lock NFSv4.x: Fix a refcount leak in nfs_callback_up_net pNFS: The client must not do I/O to the DS if it's lease has expired kernfs: don't depend on d_find_any_alias() when generating notifications powerpc/mm: Don't alias user region to other regions below PAGE_OFFSET powerpc/powernv : Drop reference added by kset_find_obj() powerpc/tm: do not use r13 for tabort_syscall tipc: move linearization of buffers to generic code lightnvm: put bio before return fscrypto: require write access to mount to set encryption policy Revert "KVM: x86: fix missed hardware breakpoints" MIPS: KVM: Check for pfn noslot case clocksource/drivers/sun4i: Clear interrupts after stopping timer in probe function fscrypto: add authorization check for setting encryption policy ext4: use __GFP_NOFAIL in ext4_free_blocks() Conflicts: arch/arm/kernel/devtree.c arch/arm64/Kconfig arch/arm64/kernel/arm64ksyms.c arch/arm64/kernel/psci.c arch/arm64/mm/fault.c drivers/android/binder.c drivers/usb/host/xhci-hub.c fs/ext4/readpage.c include/linux/mmc/core.h include/linux/mmzone.h mm/memcontrol.c net/core/filter.c net/netlink/af_netlink.c net/netlink/af_netlink.h Change-Id: I99fe7a0914e83e284b11b33185b71448a8999d1f Signed-off-by: Runmin Wang <runminw@codeaurora.org> Signed-off-by: Blagovest Kolenichev <bkolenichev@codeaurora.org>
2444 lines
66 KiB
C
2444 lines
66 KiB
C
/*
|
|
* Memory merging support.
|
|
*
|
|
* This code enables dynamic sharing of identical pages found in different
|
|
* memory areas, even if they are not shared by fork()
|
|
*
|
|
* Copyright (C) 2008-2009 Red Hat, Inc.
|
|
* Authors:
|
|
* Izik Eidus
|
|
* Andrea Arcangeli
|
|
* Chris Wright
|
|
* Hugh Dickins
|
|
*
|
|
* This work is licensed under the terms of the GNU GPL, version 2.
|
|
*/
|
|
|
|
#include <linux/errno.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/rmap.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/jhash.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/wait.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/rbtree.h>
|
|
#include <linux/memory.h>
|
|
#include <linux/mmu_notifier.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/ksm.h>
|
|
#include <linux/hashtable.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/oom.h>
|
|
#include <linux/numa.h>
|
|
#include <linux/show_mem_notifier.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
#include "internal.h"
|
|
|
|
#ifdef CONFIG_NUMA
|
|
#define NUMA(x) (x)
|
|
#define DO_NUMA(x) do { (x); } while (0)
|
|
#else
|
|
#define NUMA(x) (0)
|
|
#define DO_NUMA(x) do { } while (0)
|
|
#endif
|
|
|
|
/*
|
|
* A few notes about the KSM scanning process,
|
|
* to make it easier to understand the data structures below:
|
|
*
|
|
* In order to reduce excessive scanning, KSM sorts the memory pages by their
|
|
* contents into a data structure that holds pointers to the pages' locations.
|
|
*
|
|
* Since the contents of the pages may change at any moment, KSM cannot just
|
|
* insert the pages into a normal sorted tree and expect it to find anything.
|
|
* Therefore KSM uses two data structures - the stable and the unstable tree.
|
|
*
|
|
* The stable tree holds pointers to all the merged pages (ksm pages), sorted
|
|
* by their contents. Because each such page is write-protected, searching on
|
|
* this tree is fully assured to be working (except when pages are unmapped),
|
|
* and therefore this tree is called the stable tree.
|
|
*
|
|
* In addition to the stable tree, KSM uses a second data structure called the
|
|
* unstable tree: this tree holds pointers to pages which have been found to
|
|
* be "unchanged for a period of time". The unstable tree sorts these pages
|
|
* by their contents, but since they are not write-protected, KSM cannot rely
|
|
* upon the unstable tree to work correctly - the unstable tree is liable to
|
|
* be corrupted as its contents are modified, and so it is called unstable.
|
|
*
|
|
* KSM solves this problem by several techniques:
|
|
*
|
|
* 1) The unstable tree is flushed every time KSM completes scanning all
|
|
* memory areas, and then the tree is rebuilt again from the beginning.
|
|
* 2) KSM will only insert into the unstable tree, pages whose hash value
|
|
* has not changed since the previous scan of all memory areas.
|
|
* 3) The unstable tree is a RedBlack Tree - so its balancing is based on the
|
|
* colors of the nodes and not on their contents, assuring that even when
|
|
* the tree gets "corrupted" it won't get out of balance, so scanning time
|
|
* remains the same (also, searching and inserting nodes in an rbtree uses
|
|
* the same algorithm, so we have no overhead when we flush and rebuild).
|
|
* 4) KSM never flushes the stable tree, which means that even if it were to
|
|
* take 10 attempts to find a page in the unstable tree, once it is found,
|
|
* it is secured in the stable tree. (When we scan a new page, we first
|
|
* compare it against the stable tree, and then against the unstable tree.)
|
|
*
|
|
* If the merge_across_nodes tunable is unset, then KSM maintains multiple
|
|
* stable trees and multiple unstable trees: one of each for each NUMA node.
|
|
*/
|
|
|
|
/**
|
|
* struct mm_slot - ksm information per mm that is being scanned
|
|
* @link: link to the mm_slots hash list
|
|
* @mm_list: link into the mm_slots list, rooted in ksm_mm_head
|
|
* @rmap_list: head for this mm_slot's singly-linked list of rmap_items
|
|
* @mm: the mm that this information is valid for
|
|
*/
|
|
struct mm_slot {
|
|
struct hlist_node link;
|
|
struct list_head mm_list;
|
|
struct rmap_item *rmap_list;
|
|
struct mm_struct *mm;
|
|
};
|
|
|
|
/**
|
|
* struct ksm_scan - cursor for scanning
|
|
* @mm_slot: the current mm_slot we are scanning
|
|
* @address: the next address inside that to be scanned
|
|
* @rmap_list: link to the next rmap to be scanned in the rmap_list
|
|
* @seqnr: count of completed full scans (needed when removing unstable node)
|
|
*
|
|
* There is only the one ksm_scan instance of this cursor structure.
|
|
*/
|
|
struct ksm_scan {
|
|
struct mm_slot *mm_slot;
|
|
unsigned long address;
|
|
struct rmap_item **rmap_list;
|
|
unsigned long seqnr;
|
|
};
|
|
|
|
/**
|
|
* struct stable_node - node of the stable rbtree
|
|
* @node: rb node of this ksm page in the stable tree
|
|
* @head: (overlaying parent) &migrate_nodes indicates temporarily on that list
|
|
* @list: linked into migrate_nodes, pending placement in the proper node tree
|
|
* @hlist: hlist head of rmap_items using this ksm page
|
|
* @kpfn: page frame number of this ksm page (perhaps temporarily on wrong nid)
|
|
* @nid: NUMA node id of stable tree in which linked (may not match kpfn)
|
|
*/
|
|
struct stable_node {
|
|
union {
|
|
struct rb_node node; /* when node of stable tree */
|
|
struct { /* when listed for migration */
|
|
struct list_head *head;
|
|
struct list_head list;
|
|
};
|
|
};
|
|
struct hlist_head hlist;
|
|
unsigned long kpfn;
|
|
#ifdef CONFIG_NUMA
|
|
int nid;
|
|
#endif
|
|
};
|
|
|
|
/**
|
|
* struct rmap_item - reverse mapping item for virtual addresses
|
|
* @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
|
|
* @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
|
|
* @nid: NUMA node id of unstable tree in which linked (may not match page)
|
|
* @mm: the memory structure this rmap_item is pointing into
|
|
* @address: the virtual address this rmap_item tracks (+ flags in low bits)
|
|
* @oldchecksum: previous checksum of the page at that virtual address
|
|
* @node: rb node of this rmap_item in the unstable tree
|
|
* @head: pointer to stable_node heading this list in the stable tree
|
|
* @hlist: link into hlist of rmap_items hanging off that stable_node
|
|
*/
|
|
struct rmap_item {
|
|
struct rmap_item *rmap_list;
|
|
union {
|
|
struct anon_vma *anon_vma; /* when stable */
|
|
#ifdef CONFIG_NUMA
|
|
int nid; /* when node of unstable tree */
|
|
#endif
|
|
};
|
|
struct mm_struct *mm;
|
|
unsigned long address; /* + low bits used for flags below */
|
|
unsigned int oldchecksum; /* when unstable */
|
|
union {
|
|
struct rb_node node; /* when node of unstable tree */
|
|
struct { /* when listed from stable tree */
|
|
struct stable_node *head;
|
|
struct hlist_node hlist;
|
|
};
|
|
};
|
|
};
|
|
|
|
#define SEQNR_MASK 0x0ff /* low bits of unstable tree seqnr */
|
|
#define UNSTABLE_FLAG 0x100 /* is a node of the unstable tree */
|
|
#define STABLE_FLAG 0x200 /* is listed from the stable tree */
|
|
|
|
/* The stable and unstable tree heads */
|
|
static struct rb_root one_stable_tree[1] = { RB_ROOT };
|
|
static struct rb_root one_unstable_tree[1] = { RB_ROOT };
|
|
static struct rb_root *root_stable_tree = one_stable_tree;
|
|
static struct rb_root *root_unstable_tree = one_unstable_tree;
|
|
|
|
/* Recently migrated nodes of stable tree, pending proper placement */
|
|
static LIST_HEAD(migrate_nodes);
|
|
|
|
#define MM_SLOTS_HASH_BITS 10
|
|
static DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
|
|
|
|
static struct mm_slot ksm_mm_head = {
|
|
.mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
|
|
};
|
|
static struct ksm_scan ksm_scan = {
|
|
.mm_slot = &ksm_mm_head,
|
|
};
|
|
|
|
static struct kmem_cache *rmap_item_cache;
|
|
static struct kmem_cache *stable_node_cache;
|
|
static struct kmem_cache *mm_slot_cache;
|
|
|
|
/* The number of nodes in the stable tree */
|
|
static unsigned long ksm_pages_shared;
|
|
|
|
/* The number of page slots additionally sharing those nodes */
|
|
static unsigned long ksm_pages_sharing;
|
|
|
|
/* The number of nodes in the unstable tree */
|
|
static unsigned long ksm_pages_unshared;
|
|
|
|
/* The number of rmap_items in use: to calculate pages_volatile */
|
|
static unsigned long ksm_rmap_items;
|
|
|
|
/* Number of pages ksmd should scan in one batch */
|
|
static unsigned int ksm_thread_pages_to_scan = 100;
|
|
|
|
/* Milliseconds ksmd should sleep between batches */
|
|
static unsigned int ksm_thread_sleep_millisecs = 20;
|
|
|
|
/* Boolean to indicate whether to use deferred timer or not */
|
|
static bool use_deferred_timer;
|
|
|
|
#ifdef CONFIG_NUMA
|
|
/* Zeroed when merging across nodes is not allowed */
|
|
static unsigned int ksm_merge_across_nodes = 1;
|
|
static int ksm_nr_node_ids = 1;
|
|
#else
|
|
#define ksm_merge_across_nodes 1U
|
|
#define ksm_nr_node_ids 1
|
|
#endif
|
|
|
|
#define KSM_RUN_STOP 0
|
|
#define KSM_RUN_MERGE 1
|
|
#define KSM_RUN_UNMERGE 2
|
|
#define KSM_RUN_OFFLINE 4
|
|
static unsigned long ksm_run = KSM_RUN_MERGE;
|
|
static void wait_while_offlining(void);
|
|
|
|
static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
|
|
static DEFINE_MUTEX(ksm_thread_mutex);
|
|
static DEFINE_SPINLOCK(ksm_mmlist_lock);
|
|
|
|
#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
|
|
sizeof(struct __struct), __alignof__(struct __struct),\
|
|
(__flags), NULL)
|
|
|
|
static int ksm_show_mem_notifier(struct notifier_block *nb,
|
|
unsigned long action,
|
|
void *data)
|
|
{
|
|
pr_info("ksm_pages_sharing: %lu\n", ksm_pages_sharing);
|
|
pr_info("ksm_pages_shared: %lu\n", ksm_pages_shared);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct notifier_block ksm_show_mem_notifier_block = {
|
|
.notifier_call = ksm_show_mem_notifier,
|
|
};
|
|
|
|
static int __init ksm_slab_init(void)
|
|
{
|
|
rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
|
|
if (!rmap_item_cache)
|
|
goto out;
|
|
|
|
stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
|
|
if (!stable_node_cache)
|
|
goto out_free1;
|
|
|
|
mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
|
|
if (!mm_slot_cache)
|
|
goto out_free2;
|
|
|
|
return 0;
|
|
|
|
out_free2:
|
|
kmem_cache_destroy(stable_node_cache);
|
|
out_free1:
|
|
kmem_cache_destroy(rmap_item_cache);
|
|
out:
|
|
return -ENOMEM;
|
|
}
|
|
|
|
static void __init ksm_slab_free(void)
|
|
{
|
|
kmem_cache_destroy(mm_slot_cache);
|
|
kmem_cache_destroy(stable_node_cache);
|
|
kmem_cache_destroy(rmap_item_cache);
|
|
mm_slot_cache = NULL;
|
|
}
|
|
|
|
static inline struct rmap_item *alloc_rmap_item(void)
|
|
{
|
|
struct rmap_item *rmap_item;
|
|
|
|
rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
|
|
__GFP_NORETRY | __GFP_NOWARN);
|
|
if (rmap_item)
|
|
ksm_rmap_items++;
|
|
return rmap_item;
|
|
}
|
|
|
|
static inline void free_rmap_item(struct rmap_item *rmap_item)
|
|
{
|
|
ksm_rmap_items--;
|
|
rmap_item->mm = NULL; /* debug safety */
|
|
kmem_cache_free(rmap_item_cache, rmap_item);
|
|
}
|
|
|
|
static inline struct stable_node *alloc_stable_node(void)
|
|
{
|
|
return kmem_cache_alloc(stable_node_cache, GFP_KERNEL);
|
|
}
|
|
|
|
static inline void free_stable_node(struct stable_node *stable_node)
|
|
{
|
|
kmem_cache_free(stable_node_cache, stable_node);
|
|
}
|
|
|
|
static inline struct mm_slot *alloc_mm_slot(void)
|
|
{
|
|
if (!mm_slot_cache) /* initialization failed */
|
|
return NULL;
|
|
return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
|
|
}
|
|
|
|
static inline void free_mm_slot(struct mm_slot *mm_slot)
|
|
{
|
|
kmem_cache_free(mm_slot_cache, mm_slot);
|
|
}
|
|
|
|
static struct mm_slot *get_mm_slot(struct mm_struct *mm)
|
|
{
|
|
struct mm_slot *slot;
|
|
|
|
hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)
|
|
if (slot->mm == mm)
|
|
return slot;
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void insert_to_mm_slots_hash(struct mm_struct *mm,
|
|
struct mm_slot *mm_slot)
|
|
{
|
|
mm_slot->mm = mm;
|
|
hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);
|
|
}
|
|
|
|
/*
|
|
* ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
|
|
* page tables after it has passed through ksm_exit() - which, if necessary,
|
|
* takes mmap_sem briefly to serialize against them. ksm_exit() does not set
|
|
* a special flag: they can just back out as soon as mm_users goes to zero.
|
|
* ksm_test_exit() is used throughout to make this test for exit: in some
|
|
* places for correctness, in some places just to avoid unnecessary work.
|
|
*/
|
|
static inline bool ksm_test_exit(struct mm_struct *mm)
|
|
{
|
|
return atomic_read(&mm->mm_users) == 0;
|
|
}
|
|
|
|
/*
|
|
* We use break_ksm to break COW on a ksm page: it's a stripped down
|
|
*
|
|
* if (get_user_pages(current, mm, addr, 1, 1, 1, &page, NULL) == 1)
|
|
* put_page(page);
|
|
*
|
|
* but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
|
|
* in case the application has unmapped and remapped mm,addr meanwhile.
|
|
* Could a ksm page appear anywhere else? Actually yes, in a VM_PFNMAP
|
|
* mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
|
|
*/
|
|
static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
|
|
{
|
|
struct page *page;
|
|
int ret = 0;
|
|
|
|
do {
|
|
cond_resched();
|
|
page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION);
|
|
if (IS_ERR_OR_NULL(page))
|
|
break;
|
|
if (PageKsm(page))
|
|
ret = handle_mm_fault(vma->vm_mm, vma, addr,
|
|
FAULT_FLAG_WRITE);
|
|
else
|
|
ret = VM_FAULT_WRITE;
|
|
put_page(page);
|
|
} while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM)));
|
|
/*
|
|
* We must loop because handle_mm_fault() may back out if there's
|
|
* any difficulty e.g. if pte accessed bit gets updated concurrently.
|
|
*
|
|
* VM_FAULT_WRITE is what we have been hoping for: it indicates that
|
|
* COW has been broken, even if the vma does not permit VM_WRITE;
|
|
* but note that a concurrent fault might break PageKsm for us.
|
|
*
|
|
* VM_FAULT_SIGBUS could occur if we race with truncation of the
|
|
* backing file, which also invalidates anonymous pages: that's
|
|
* okay, that truncation will have unmapped the PageKsm for us.
|
|
*
|
|
* VM_FAULT_OOM: at the time of writing (late July 2009), setting
|
|
* aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
|
|
* current task has TIF_MEMDIE set, and will be OOM killed on return
|
|
* to user; and ksmd, having no mm, would never be chosen for that.
|
|
*
|
|
* But if the mm is in a limited mem_cgroup, then the fault may fail
|
|
* with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
|
|
* even ksmd can fail in this way - though it's usually breaking ksm
|
|
* just to undo a merge it made a moment before, so unlikely to oom.
|
|
*
|
|
* That's a pity: we might therefore have more kernel pages allocated
|
|
* than we're counting as nodes in the stable tree; but ksm_do_scan
|
|
* will retry to break_cow on each pass, so should recover the page
|
|
* in due course. The important thing is to not let VM_MERGEABLE
|
|
* be cleared while any such pages might remain in the area.
|
|
*/
|
|
return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
|
|
}
|
|
|
|
static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
|
|
unsigned long addr)
|
|
{
|
|
struct vm_area_struct *vma;
|
|
if (ksm_test_exit(mm))
|
|
return NULL;
|
|
vma = find_vma(mm, addr);
|
|
if (!vma || vma->vm_start > addr)
|
|
return NULL;
|
|
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
|
|
return NULL;
|
|
return vma;
|
|
}
|
|
|
|
static void break_cow(struct rmap_item *rmap_item)
|
|
{
|
|
struct mm_struct *mm = rmap_item->mm;
|
|
unsigned long addr = rmap_item->address;
|
|
struct vm_area_struct *vma;
|
|
|
|
/*
|
|
* It is not an accident that whenever we want to break COW
|
|
* to undo, we also need to drop a reference to the anon_vma.
|
|
*/
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
|
|
down_read(&mm->mmap_sem);
|
|
vma = find_mergeable_vma(mm, addr);
|
|
if (vma)
|
|
break_ksm(vma, addr);
|
|
up_read(&mm->mmap_sem);
|
|
}
|
|
|
|
static struct page *page_trans_compound_anon(struct page *page)
|
|
{
|
|
if (PageTransCompound(page)) {
|
|
struct page *head = compound_head(page);
|
|
/*
|
|
* head may actually be splitted and freed from under
|
|
* us but it's ok here.
|
|
*/
|
|
if (PageAnon(head))
|
|
return head;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static struct page *get_mergeable_page(struct rmap_item *rmap_item)
|
|
{
|
|
struct mm_struct *mm = rmap_item->mm;
|
|
unsigned long addr = rmap_item->address;
|
|
struct vm_area_struct *vma;
|
|
struct page *page;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
vma = find_mergeable_vma(mm, addr);
|
|
if (!vma)
|
|
goto out;
|
|
|
|
page = follow_page(vma, addr, FOLL_GET);
|
|
if (IS_ERR_OR_NULL(page))
|
|
goto out;
|
|
if (PageAnon(page) || page_trans_compound_anon(page)) {
|
|
flush_anon_page(vma, page, addr);
|
|
flush_dcache_page(page);
|
|
} else {
|
|
put_page(page);
|
|
out:
|
|
page = NULL;
|
|
}
|
|
up_read(&mm->mmap_sem);
|
|
return page;
|
|
}
|
|
|
|
/*
|
|
* This helper is used for getting right index into array of tree roots.
|
|
* When merge_across_nodes knob is set to 1, there are only two rb-trees for
|
|
* stable and unstable pages from all nodes with roots in index 0. Otherwise,
|
|
* every node has its own stable and unstable tree.
|
|
*/
|
|
static inline int get_kpfn_nid(unsigned long kpfn)
|
|
{
|
|
return ksm_merge_across_nodes ? 0 : NUMA(pfn_to_nid(kpfn));
|
|
}
|
|
|
|
static void remove_node_from_stable_tree(struct stable_node *stable_node)
|
|
{
|
|
struct rmap_item *rmap_item;
|
|
|
|
hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
|
|
if (rmap_item->hlist.next)
|
|
ksm_pages_sharing--;
|
|
else
|
|
ksm_pages_shared--;
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
rmap_item->address &= PAGE_MASK;
|
|
cond_resched();
|
|
}
|
|
|
|
if (stable_node->head == &migrate_nodes)
|
|
list_del(&stable_node->list);
|
|
else
|
|
rb_erase(&stable_node->node,
|
|
root_stable_tree + NUMA(stable_node->nid));
|
|
free_stable_node(stable_node);
|
|
}
|
|
|
|
/*
|
|
* get_ksm_page: checks if the page indicated by the stable node
|
|
* is still its ksm page, despite having held no reference to it.
|
|
* In which case we can trust the content of the page, and it
|
|
* returns the gotten page; but if the page has now been zapped,
|
|
* remove the stale node from the stable tree and return NULL.
|
|
* But beware, the stable node's page might be being migrated.
|
|
*
|
|
* You would expect the stable_node to hold a reference to the ksm page.
|
|
* But if it increments the page's count, swapping out has to wait for
|
|
* ksmd to come around again before it can free the page, which may take
|
|
* seconds or even minutes: much too unresponsive. So instead we use a
|
|
* "keyhole reference": access to the ksm page from the stable node peeps
|
|
* out through its keyhole to see if that page still holds the right key,
|
|
* pointing back to this stable node. This relies on freeing a PageAnon
|
|
* page to reset its page->mapping to NULL, and relies on no other use of
|
|
* a page to put something that might look like our key in page->mapping.
|
|
* is on its way to being freed; but it is an anomaly to bear in mind.
|
|
*/
|
|
static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it)
|
|
{
|
|
struct page *page;
|
|
void *expected_mapping;
|
|
unsigned long kpfn;
|
|
|
|
expected_mapping = (void *)((unsigned long)stable_node |
|
|
PAGE_MAPPING_KSM);
|
|
again:
|
|
kpfn = READ_ONCE(stable_node->kpfn);
|
|
page = pfn_to_page(kpfn);
|
|
|
|
/*
|
|
* page is computed from kpfn, so on most architectures reading
|
|
* page->mapping is naturally ordered after reading node->kpfn,
|
|
* but on Alpha we need to be more careful.
|
|
*/
|
|
smp_read_barrier_depends();
|
|
if (READ_ONCE(page->mapping) != expected_mapping)
|
|
goto stale;
|
|
|
|
/*
|
|
* We cannot do anything with the page while its refcount is 0.
|
|
* Usually 0 means free, or tail of a higher-order page: in which
|
|
* case this node is no longer referenced, and should be freed;
|
|
* however, it might mean that the page is under page_freeze_refs().
|
|
* The __remove_mapping() case is easy, again the node is now stale;
|
|
* but if page is swapcache in migrate_page_move_mapping(), it might
|
|
* still be our page, in which case it's essential to keep the node.
|
|
*/
|
|
while (!get_page_unless_zero(page)) {
|
|
/*
|
|
* Another check for page->mapping != expected_mapping would
|
|
* work here too. We have chosen the !PageSwapCache test to
|
|
* optimize the common case, when the page is or is about to
|
|
* be freed: PageSwapCache is cleared (under spin_lock_irq)
|
|
* in the freeze_refs section of __remove_mapping(); but Anon
|
|
* page->mapping reset to NULL later, in free_pages_prepare().
|
|
*/
|
|
if (!PageSwapCache(page))
|
|
goto stale;
|
|
cpu_relax();
|
|
}
|
|
|
|
if (READ_ONCE(page->mapping) != expected_mapping) {
|
|
put_page(page);
|
|
goto stale;
|
|
}
|
|
|
|
if (lock_it) {
|
|
lock_page(page);
|
|
if (READ_ONCE(page->mapping) != expected_mapping) {
|
|
unlock_page(page);
|
|
put_page(page);
|
|
goto stale;
|
|
}
|
|
}
|
|
return page;
|
|
|
|
stale:
|
|
/*
|
|
* We come here from above when page->mapping or !PageSwapCache
|
|
* suggests that the node is stale; but it might be under migration.
|
|
* We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(),
|
|
* before checking whether node->kpfn has been changed.
|
|
*/
|
|
smp_rmb();
|
|
if (READ_ONCE(stable_node->kpfn) != kpfn)
|
|
goto again;
|
|
remove_node_from_stable_tree(stable_node);
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Removing rmap_item from stable or unstable tree.
|
|
* This function will clean the information from the stable/unstable tree.
|
|
*/
|
|
static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
|
|
{
|
|
if (rmap_item->address & STABLE_FLAG) {
|
|
struct stable_node *stable_node;
|
|
struct page *page;
|
|
|
|
stable_node = rmap_item->head;
|
|
page = get_ksm_page(stable_node, true);
|
|
if (!page)
|
|
goto out;
|
|
|
|
hlist_del(&rmap_item->hlist);
|
|
unlock_page(page);
|
|
put_page(page);
|
|
|
|
if (!hlist_empty(&stable_node->hlist))
|
|
ksm_pages_sharing--;
|
|
else
|
|
ksm_pages_shared--;
|
|
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
rmap_item->address &= PAGE_MASK;
|
|
|
|
} else if (rmap_item->address & UNSTABLE_FLAG) {
|
|
unsigned char age;
|
|
/*
|
|
* Usually ksmd can and must skip the rb_erase, because
|
|
* root_unstable_tree was already reset to RB_ROOT.
|
|
* But be careful when an mm is exiting: do the rb_erase
|
|
* if this rmap_item was inserted by this scan, rather
|
|
* than left over from before.
|
|
*/
|
|
age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
|
|
BUG_ON(age > 1);
|
|
if (!age)
|
|
rb_erase(&rmap_item->node,
|
|
root_unstable_tree + NUMA(rmap_item->nid));
|
|
ksm_pages_unshared--;
|
|
rmap_item->address &= PAGE_MASK;
|
|
}
|
|
out:
|
|
cond_resched(); /* we're called from many long loops */
|
|
}
|
|
|
|
static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
|
|
struct rmap_item **rmap_list)
|
|
{
|
|
while (*rmap_list) {
|
|
struct rmap_item *rmap_item = *rmap_list;
|
|
*rmap_list = rmap_item->rmap_list;
|
|
remove_rmap_item_from_tree(rmap_item);
|
|
free_rmap_item(rmap_item);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Though it's very tempting to unmerge rmap_items from stable tree rather
|
|
* than check every pte of a given vma, the locking doesn't quite work for
|
|
* that - an rmap_item is assigned to the stable tree after inserting ksm
|
|
* page and upping mmap_sem. Nor does it fit with the way we skip dup'ing
|
|
* rmap_items from parent to child at fork time (so as not to waste time
|
|
* if exit comes before the next scan reaches it).
|
|
*
|
|
* Similarly, although we'd like to remove rmap_items (so updating counts
|
|
* and freeing memory) when unmerging an area, it's easier to leave that
|
|
* to the next pass of ksmd - consider, for example, how ksmd might be
|
|
* in cmp_and_merge_page on one of the rmap_items we would be removing.
|
|
*/
|
|
static int unmerge_ksm_pages(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long addr;
|
|
int err = 0;
|
|
|
|
for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
|
|
if (ksm_test_exit(vma->vm_mm))
|
|
break;
|
|
if (signal_pending(current))
|
|
err = -ERESTARTSYS;
|
|
else
|
|
err = break_ksm(vma, addr);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
#ifdef CONFIG_SYSFS
|
|
/*
|
|
* Only called through the sysfs control interface:
|
|
*/
|
|
static int remove_stable_node(struct stable_node *stable_node)
|
|
{
|
|
struct page *page;
|
|
int err;
|
|
|
|
page = get_ksm_page(stable_node, true);
|
|
if (!page) {
|
|
/*
|
|
* get_ksm_page did remove_node_from_stable_tree itself.
|
|
*/
|
|
return 0;
|
|
}
|
|
|
|
if (WARN_ON_ONCE(page_mapped(page))) {
|
|
/*
|
|
* This should not happen: but if it does, just refuse to let
|
|
* merge_across_nodes be switched - there is no need to panic.
|
|
*/
|
|
err = -EBUSY;
|
|
} else {
|
|
/*
|
|
* The stable node did not yet appear stale to get_ksm_page(),
|
|
* since that allows for an unmapped ksm page to be recognized
|
|
* right up until it is freed; but the node is safe to remove.
|
|
* This page might be in a pagevec waiting to be freed,
|
|
* or it might be PageSwapCache (perhaps under writeback),
|
|
* or it might have been removed from swapcache a moment ago.
|
|
*/
|
|
set_page_stable_node(page, NULL);
|
|
remove_node_from_stable_tree(stable_node);
|
|
err = 0;
|
|
}
|
|
|
|
unlock_page(page);
|
|
put_page(page);
|
|
return err;
|
|
}
|
|
|
|
static int remove_all_stable_nodes(void)
|
|
{
|
|
struct stable_node *stable_node;
|
|
struct list_head *this, *next;
|
|
int nid;
|
|
int err = 0;
|
|
|
|
for (nid = 0; nid < ksm_nr_node_ids; nid++) {
|
|
while (root_stable_tree[nid].rb_node) {
|
|
stable_node = rb_entry(root_stable_tree[nid].rb_node,
|
|
struct stable_node, node);
|
|
if (remove_stable_node(stable_node)) {
|
|
err = -EBUSY;
|
|
break; /* proceed to next nid */
|
|
}
|
|
cond_resched();
|
|
}
|
|
}
|
|
list_for_each_safe(this, next, &migrate_nodes) {
|
|
stable_node = list_entry(this, struct stable_node, list);
|
|
if (remove_stable_node(stable_node))
|
|
err = -EBUSY;
|
|
cond_resched();
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static int unmerge_and_remove_all_rmap_items(void)
|
|
{
|
|
struct mm_slot *mm_slot;
|
|
struct mm_struct *mm;
|
|
struct vm_area_struct *vma;
|
|
int err = 0;
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
|
|
struct mm_slot, mm_list);
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
|
|
for (mm_slot = ksm_scan.mm_slot;
|
|
mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
|
|
mm = mm_slot->mm;
|
|
down_read(&mm->mmap_sem);
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
|
if (ksm_test_exit(mm))
|
|
break;
|
|
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
|
|
continue;
|
|
err = unmerge_ksm_pages(vma,
|
|
vma->vm_start, vma->vm_end);
|
|
if (err)
|
|
goto error;
|
|
}
|
|
|
|
remove_trailing_rmap_items(mm_slot, &mm_slot->rmap_list);
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
|
|
struct mm_slot, mm_list);
|
|
if (ksm_test_exit(mm)) {
|
|
hash_del(&mm_slot->link);
|
|
list_del(&mm_slot->mm_list);
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
|
|
free_mm_slot(mm_slot);
|
|
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
|
|
up_read(&mm->mmap_sem);
|
|
mmdrop(mm);
|
|
} else {
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
up_read(&mm->mmap_sem);
|
|
}
|
|
}
|
|
|
|
/* Clean up stable nodes, but don't worry if some are still busy */
|
|
remove_all_stable_nodes();
|
|
ksm_scan.seqnr = 0;
|
|
return 0;
|
|
|
|
error:
|
|
up_read(&mm->mmap_sem);
|
|
spin_lock(&ksm_mmlist_lock);
|
|
ksm_scan.mm_slot = &ksm_mm_head;
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
return err;
|
|
}
|
|
#endif /* CONFIG_SYSFS */
|
|
|
|
static u32 calc_checksum(struct page *page)
|
|
{
|
|
u32 checksum;
|
|
void *addr = kmap_atomic(page);
|
|
checksum = jhash2(addr, PAGE_SIZE / 4, 17);
|
|
kunmap_atomic(addr);
|
|
return checksum;
|
|
}
|
|
|
|
static int memcmp_pages(struct page *page1, struct page *page2)
|
|
{
|
|
char *addr1, *addr2;
|
|
int ret;
|
|
|
|
addr1 = kmap_atomic(page1);
|
|
addr2 = kmap_atomic(page2);
|
|
ret = memcmp(addr1, addr2, PAGE_SIZE);
|
|
kunmap_atomic(addr2);
|
|
kunmap_atomic(addr1);
|
|
return ret;
|
|
}
|
|
|
|
static inline int pages_identical(struct page *page1, struct page *page2)
|
|
{
|
|
return !memcmp_pages(page1, page2);
|
|
}
|
|
|
|
static int write_protect_page(struct vm_area_struct *vma, struct page *page,
|
|
pte_t *orig_pte)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
unsigned long addr;
|
|
pte_t *ptep;
|
|
spinlock_t *ptl;
|
|
int swapped;
|
|
int err = -EFAULT;
|
|
unsigned long mmun_start; /* For mmu_notifiers */
|
|
unsigned long mmun_end; /* For mmu_notifiers */
|
|
|
|
addr = page_address_in_vma(page, vma);
|
|
if (addr == -EFAULT)
|
|
goto out;
|
|
|
|
BUG_ON(PageTransCompound(page));
|
|
|
|
mmun_start = addr;
|
|
mmun_end = addr + PAGE_SIZE;
|
|
mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
|
|
|
|
ptep = page_check_address(page, mm, addr, &ptl, 0);
|
|
if (!ptep)
|
|
goto out_mn;
|
|
|
|
if (pte_write(*ptep) || pte_dirty(*ptep)) {
|
|
pte_t entry;
|
|
|
|
swapped = PageSwapCache(page);
|
|
flush_cache_page(vma, addr, page_to_pfn(page));
|
|
/*
|
|
* Ok this is tricky, when get_user_pages_fast() run it doesn't
|
|
* take any lock, therefore the check that we are going to make
|
|
* with the pagecount against the mapcount is racey and
|
|
* O_DIRECT can happen right after the check.
|
|
* So we clear the pte and flush the tlb before the check
|
|
* this assure us that no O_DIRECT can happen after the check
|
|
* or in the middle of the check.
|
|
*/
|
|
entry = ptep_clear_flush_notify(vma, addr, ptep);
|
|
/*
|
|
* Check that no O_DIRECT or similar I/O is in progress on the
|
|
* page
|
|
*/
|
|
if (page_mapcount(page) + 1 + swapped != page_count(page)) {
|
|
set_pte_at(mm, addr, ptep, entry);
|
|
goto out_unlock;
|
|
}
|
|
if (pte_dirty(entry))
|
|
set_page_dirty(page);
|
|
entry = pte_mkclean(pte_wrprotect(entry));
|
|
set_pte_at_notify(mm, addr, ptep, entry);
|
|
}
|
|
*orig_pte = *ptep;
|
|
err = 0;
|
|
|
|
out_unlock:
|
|
pte_unmap_unlock(ptep, ptl);
|
|
out_mn:
|
|
mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
|
|
out:
|
|
return err;
|
|
}
|
|
|
|
/**
|
|
* replace_page - replace page in vma by new ksm page
|
|
* @vma: vma that holds the pte pointing to page
|
|
* @page: the page we are replacing by kpage
|
|
* @kpage: the ksm page we replace page by
|
|
* @orig_pte: the original value of the pte
|
|
*
|
|
* Returns 0 on success, -EFAULT on failure.
|
|
*/
|
|
static int replace_page(struct vm_area_struct *vma, struct page *page,
|
|
struct page *kpage, pte_t orig_pte)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
pmd_t *pmd;
|
|
pte_t *ptep;
|
|
spinlock_t *ptl;
|
|
unsigned long addr;
|
|
int err = -EFAULT;
|
|
unsigned long mmun_start; /* For mmu_notifiers */
|
|
unsigned long mmun_end; /* For mmu_notifiers */
|
|
|
|
addr = page_address_in_vma(page, vma);
|
|
if (addr == -EFAULT)
|
|
goto out;
|
|
|
|
pmd = mm_find_pmd(mm, addr);
|
|
if (!pmd)
|
|
goto out;
|
|
|
|
mmun_start = addr;
|
|
mmun_end = addr + PAGE_SIZE;
|
|
mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
|
|
|
|
ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
|
|
if (!pte_same(*ptep, orig_pte)) {
|
|
pte_unmap_unlock(ptep, ptl);
|
|
goto out_mn;
|
|
}
|
|
|
|
get_page(kpage);
|
|
page_add_anon_rmap(kpage, vma, addr);
|
|
|
|
flush_cache_page(vma, addr, pte_pfn(*ptep));
|
|
ptep_clear_flush_notify(vma, addr, ptep);
|
|
set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
|
|
|
|
page_remove_rmap(page);
|
|
if (!page_mapped(page))
|
|
try_to_free_swap(page);
|
|
put_page(page);
|
|
|
|
pte_unmap_unlock(ptep, ptl);
|
|
err = 0;
|
|
out_mn:
|
|
mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
|
|
out:
|
|
return err;
|
|
}
|
|
|
|
static int page_trans_compound_anon_split(struct page *page)
|
|
{
|
|
int ret = 0;
|
|
struct page *transhuge_head = page_trans_compound_anon(page);
|
|
if (transhuge_head) {
|
|
/* Get the reference on the head to split it. */
|
|
if (get_page_unless_zero(transhuge_head)) {
|
|
/*
|
|
* Recheck we got the reference while the head
|
|
* was still anonymous.
|
|
*/
|
|
if (PageAnon(transhuge_head))
|
|
ret = split_huge_page(transhuge_head);
|
|
else
|
|
/*
|
|
* Retry later if split_huge_page run
|
|
* from under us.
|
|
*/
|
|
ret = 1;
|
|
put_page(transhuge_head);
|
|
} else
|
|
/* Retry later if split_huge_page run from under us. */
|
|
ret = 1;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* try_to_merge_one_page - take two pages and merge them into one
|
|
* @vma: the vma that holds the pte pointing to page
|
|
* @page: the PageAnon page that we want to replace with kpage
|
|
* @kpage: the PageKsm page that we want to map instead of page,
|
|
* or NULL the first time when we want to use page as kpage.
|
|
*
|
|
* This function returns 0 if the pages were merged, -EFAULT otherwise.
|
|
*/
|
|
static int try_to_merge_one_page(struct vm_area_struct *vma,
|
|
struct page *page, struct page *kpage)
|
|
{
|
|
pte_t orig_pte = __pte(0);
|
|
int err = -EFAULT;
|
|
|
|
if (page == kpage) /* ksm page forked */
|
|
return 0;
|
|
|
|
if (PageTransCompound(page) && page_trans_compound_anon_split(page))
|
|
goto out;
|
|
BUG_ON(PageTransCompound(page));
|
|
if (!PageAnon(page))
|
|
goto out;
|
|
|
|
/*
|
|
* We need the page lock to read a stable PageSwapCache in
|
|
* write_protect_page(). We use trylock_page() instead of
|
|
* lock_page() because we don't want to wait here - we
|
|
* prefer to continue scanning and merging different pages,
|
|
* then come back to this page when it is unlocked.
|
|
*/
|
|
if (!trylock_page(page))
|
|
goto out;
|
|
/*
|
|
* If this anonymous page is mapped only here, its pte may need
|
|
* to be write-protected. If it's mapped elsewhere, all of its
|
|
* ptes are necessarily already write-protected. But in either
|
|
* case, we need to lock and check page_count is not raised.
|
|
*/
|
|
if (write_protect_page(vma, page, &orig_pte) == 0) {
|
|
if (!kpage) {
|
|
/*
|
|
* While we hold page lock, upgrade page from
|
|
* PageAnon+anon_vma to PageKsm+NULL stable_node:
|
|
* stable_tree_insert() will update stable_node.
|
|
*/
|
|
set_page_stable_node(page, NULL);
|
|
mark_page_accessed(page);
|
|
err = 0;
|
|
} else if (pages_identical(page, kpage))
|
|
err = replace_page(vma, page, kpage, orig_pte);
|
|
}
|
|
|
|
if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
|
|
munlock_vma_page(page);
|
|
if (!PageMlocked(kpage)) {
|
|
unlock_page(page);
|
|
lock_page(kpage);
|
|
mlock_vma_page(kpage);
|
|
page = kpage; /* for final unlock */
|
|
}
|
|
}
|
|
|
|
unlock_page(page);
|
|
out:
|
|
return err;
|
|
}
|
|
|
|
/*
|
|
* try_to_merge_with_ksm_page - like try_to_merge_two_pages,
|
|
* but no new kernel page is allocated: kpage must already be a ksm page.
|
|
*
|
|
* This function returns 0 if the pages were merged, -EFAULT otherwise.
|
|
*/
|
|
static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
|
|
struct page *page, struct page *kpage)
|
|
{
|
|
struct mm_struct *mm = rmap_item->mm;
|
|
struct vm_area_struct *vma;
|
|
int err = -EFAULT;
|
|
|
|
down_read(&mm->mmap_sem);
|
|
vma = find_mergeable_vma(mm, rmap_item->address);
|
|
if (!vma)
|
|
goto out;
|
|
|
|
err = try_to_merge_one_page(vma, page, kpage);
|
|
if (err)
|
|
goto out;
|
|
|
|
/* Unstable nid is in union with stable anon_vma: remove first */
|
|
remove_rmap_item_from_tree(rmap_item);
|
|
|
|
/* Must get reference to anon_vma while still holding mmap_sem */
|
|
rmap_item->anon_vma = vma->anon_vma;
|
|
get_anon_vma(vma->anon_vma);
|
|
out:
|
|
up_read(&mm->mmap_sem);
|
|
return err;
|
|
}
|
|
|
|
/*
|
|
* try_to_merge_two_pages - take two identical pages and prepare them
|
|
* to be merged into one page.
|
|
*
|
|
* This function returns the kpage if we successfully merged two identical
|
|
* pages into one ksm page, NULL otherwise.
|
|
*
|
|
* Note that this function upgrades page to ksm page: if one of the pages
|
|
* is already a ksm page, try_to_merge_with_ksm_page should be used.
|
|
*/
|
|
static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
|
|
struct page *page,
|
|
struct rmap_item *tree_rmap_item,
|
|
struct page *tree_page)
|
|
{
|
|
int err;
|
|
|
|
err = try_to_merge_with_ksm_page(rmap_item, page, NULL);
|
|
if (!err) {
|
|
err = try_to_merge_with_ksm_page(tree_rmap_item,
|
|
tree_page, page);
|
|
/*
|
|
* If that fails, we have a ksm page with only one pte
|
|
* pointing to it: so break it.
|
|
*/
|
|
if (err)
|
|
break_cow(rmap_item);
|
|
}
|
|
return err ? NULL : page;
|
|
}
|
|
|
|
/*
|
|
* stable_tree_search - search for page inside the stable tree
|
|
*
|
|
* This function checks if there is a page inside the stable tree
|
|
* with identical content to the page that we are scanning right now.
|
|
*
|
|
* This function returns the stable tree node of identical content if found,
|
|
* NULL otherwise.
|
|
*/
|
|
static struct page *stable_tree_search(struct page *page)
|
|
{
|
|
int nid;
|
|
struct rb_root *root;
|
|
struct rb_node **new;
|
|
struct rb_node *parent;
|
|
struct stable_node *stable_node;
|
|
struct stable_node *page_node;
|
|
|
|
page_node = page_stable_node(page);
|
|
if (page_node && page_node->head != &migrate_nodes) {
|
|
/* ksm page forked */
|
|
get_page(page);
|
|
return page;
|
|
}
|
|
|
|
nid = get_kpfn_nid(page_to_pfn(page));
|
|
root = root_stable_tree + nid;
|
|
again:
|
|
new = &root->rb_node;
|
|
parent = NULL;
|
|
|
|
while (*new) {
|
|
struct page *tree_page;
|
|
int ret;
|
|
|
|
cond_resched();
|
|
stable_node = rb_entry(*new, struct stable_node, node);
|
|
tree_page = get_ksm_page(stable_node, false);
|
|
if (!tree_page) {
|
|
/*
|
|
* If we walked over a stale stable_node,
|
|
* get_ksm_page() will call rb_erase() and it
|
|
* may rebalance the tree from under us. So
|
|
* restart the search from scratch. Returning
|
|
* NULL would be safe too, but we'd generate
|
|
* false negative insertions just because some
|
|
* stable_node was stale.
|
|
*/
|
|
goto again;
|
|
}
|
|
|
|
ret = memcmp_pages(page, tree_page);
|
|
put_page(tree_page);
|
|
|
|
parent = *new;
|
|
if (ret < 0)
|
|
new = &parent->rb_left;
|
|
else if (ret > 0)
|
|
new = &parent->rb_right;
|
|
else {
|
|
/*
|
|
* Lock and unlock the stable_node's page (which
|
|
* might already have been migrated) so that page
|
|
* migration is sure to notice its raised count.
|
|
* It would be more elegant to return stable_node
|
|
* than kpage, but that involves more changes.
|
|
*/
|
|
tree_page = get_ksm_page(stable_node, true);
|
|
if (tree_page) {
|
|
unlock_page(tree_page);
|
|
if (get_kpfn_nid(stable_node->kpfn) !=
|
|
NUMA(stable_node->nid)) {
|
|
put_page(tree_page);
|
|
goto replace;
|
|
}
|
|
return tree_page;
|
|
}
|
|
/*
|
|
* There is now a place for page_node, but the tree may
|
|
* have been rebalanced, so re-evaluate parent and new.
|
|
*/
|
|
if (page_node)
|
|
goto again;
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
if (!page_node)
|
|
return NULL;
|
|
|
|
list_del(&page_node->list);
|
|
DO_NUMA(page_node->nid = nid);
|
|
rb_link_node(&page_node->node, parent, new);
|
|
rb_insert_color(&page_node->node, root);
|
|
get_page(page);
|
|
return page;
|
|
|
|
replace:
|
|
if (page_node) {
|
|
list_del(&page_node->list);
|
|
DO_NUMA(page_node->nid = nid);
|
|
rb_replace_node(&stable_node->node, &page_node->node, root);
|
|
get_page(page);
|
|
} else {
|
|
rb_erase(&stable_node->node, root);
|
|
page = NULL;
|
|
}
|
|
stable_node->head = &migrate_nodes;
|
|
list_add(&stable_node->list, stable_node->head);
|
|
return page;
|
|
}
|
|
|
|
/*
|
|
* stable_tree_insert - insert stable tree node pointing to new ksm page
|
|
* into the stable tree.
|
|
*
|
|
* This function returns the stable tree node just allocated on success,
|
|
* NULL otherwise.
|
|
*/
|
|
static struct stable_node *stable_tree_insert(struct page *kpage)
|
|
{
|
|
int nid;
|
|
unsigned long kpfn;
|
|
struct rb_root *root;
|
|
struct rb_node **new;
|
|
struct rb_node *parent;
|
|
struct stable_node *stable_node;
|
|
|
|
kpfn = page_to_pfn(kpage);
|
|
nid = get_kpfn_nid(kpfn);
|
|
root = root_stable_tree + nid;
|
|
again:
|
|
parent = NULL;
|
|
new = &root->rb_node;
|
|
|
|
while (*new) {
|
|
struct page *tree_page;
|
|
int ret;
|
|
|
|
cond_resched();
|
|
stable_node = rb_entry(*new, struct stable_node, node);
|
|
tree_page = get_ksm_page(stable_node, false);
|
|
if (!tree_page) {
|
|
/*
|
|
* If we walked over a stale stable_node,
|
|
* get_ksm_page() will call rb_erase() and it
|
|
* may rebalance the tree from under us. So
|
|
* restart the search from scratch. Returning
|
|
* NULL would be safe too, but we'd generate
|
|
* false negative insertions just because some
|
|
* stable_node was stale.
|
|
*/
|
|
goto again;
|
|
}
|
|
|
|
ret = memcmp_pages(kpage, tree_page);
|
|
put_page(tree_page);
|
|
|
|
parent = *new;
|
|
if (ret < 0)
|
|
new = &parent->rb_left;
|
|
else if (ret > 0)
|
|
new = &parent->rb_right;
|
|
else {
|
|
/*
|
|
* It is not a bug that stable_tree_search() didn't
|
|
* find this node: because at that time our page was
|
|
* not yet write-protected, so may have changed since.
|
|
*/
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
stable_node = alloc_stable_node();
|
|
if (!stable_node)
|
|
return NULL;
|
|
|
|
INIT_HLIST_HEAD(&stable_node->hlist);
|
|
stable_node->kpfn = kpfn;
|
|
set_page_stable_node(kpage, stable_node);
|
|
DO_NUMA(stable_node->nid = nid);
|
|
rb_link_node(&stable_node->node, parent, new);
|
|
rb_insert_color(&stable_node->node, root);
|
|
|
|
return stable_node;
|
|
}
|
|
|
|
/*
|
|
* unstable_tree_search_insert - search for identical page,
|
|
* else insert rmap_item into the unstable tree.
|
|
*
|
|
* This function searches for a page in the unstable tree identical to the
|
|
* page currently being scanned; and if no identical page is found in the
|
|
* tree, we insert rmap_item as a new object into the unstable tree.
|
|
*
|
|
* This function returns pointer to rmap_item found to be identical
|
|
* to the currently scanned page, NULL otherwise.
|
|
*
|
|
* This function does both searching and inserting, because they share
|
|
* the same walking algorithm in an rbtree.
|
|
*/
|
|
static
|
|
struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
|
|
struct page *page,
|
|
struct page **tree_pagep)
|
|
{
|
|
struct rb_node **new;
|
|
struct rb_root *root;
|
|
struct rb_node *parent = NULL;
|
|
int nid;
|
|
|
|
nid = get_kpfn_nid(page_to_pfn(page));
|
|
root = root_unstable_tree + nid;
|
|
new = &root->rb_node;
|
|
|
|
while (*new) {
|
|
struct rmap_item *tree_rmap_item;
|
|
struct page *tree_page;
|
|
int ret;
|
|
|
|
cond_resched();
|
|
tree_rmap_item = rb_entry(*new, struct rmap_item, node);
|
|
tree_page = get_mergeable_page(tree_rmap_item);
|
|
if (!tree_page)
|
|
return NULL;
|
|
|
|
/*
|
|
* Don't substitute a ksm page for a forked page.
|
|
*/
|
|
if (page == tree_page) {
|
|
put_page(tree_page);
|
|
return NULL;
|
|
}
|
|
|
|
ret = memcmp_pages(page, tree_page);
|
|
|
|
parent = *new;
|
|
if (ret < 0) {
|
|
put_page(tree_page);
|
|
new = &parent->rb_left;
|
|
} else if (ret > 0) {
|
|
put_page(tree_page);
|
|
new = &parent->rb_right;
|
|
} else if (!ksm_merge_across_nodes &&
|
|
page_to_nid(tree_page) != nid) {
|
|
/*
|
|
* If tree_page has been migrated to another NUMA node,
|
|
* it will be flushed out and put in the right unstable
|
|
* tree next time: only merge with it when across_nodes.
|
|
*/
|
|
put_page(tree_page);
|
|
return NULL;
|
|
} else {
|
|
*tree_pagep = tree_page;
|
|
return tree_rmap_item;
|
|
}
|
|
}
|
|
|
|
rmap_item->address |= UNSTABLE_FLAG;
|
|
rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
|
|
DO_NUMA(rmap_item->nid = nid);
|
|
rb_link_node(&rmap_item->node, parent, new);
|
|
rb_insert_color(&rmap_item->node, root);
|
|
|
|
ksm_pages_unshared++;
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* stable_tree_append - add another rmap_item to the linked list of
|
|
* rmap_items hanging off a given node of the stable tree, all sharing
|
|
* the same ksm page.
|
|
*/
|
|
static void stable_tree_append(struct rmap_item *rmap_item,
|
|
struct stable_node *stable_node)
|
|
{
|
|
rmap_item->head = stable_node;
|
|
rmap_item->address |= STABLE_FLAG;
|
|
hlist_add_head(&rmap_item->hlist, &stable_node->hlist);
|
|
|
|
if (rmap_item->hlist.next)
|
|
ksm_pages_sharing++;
|
|
else
|
|
ksm_pages_shared++;
|
|
}
|
|
|
|
/*
|
|
* cmp_and_merge_page - first see if page can be merged into the stable tree;
|
|
* if not, compare checksum to previous and if it's the same, see if page can
|
|
* be inserted into the unstable tree, or merged with a page already there and
|
|
* both transferred to the stable tree.
|
|
*
|
|
* @page: the page that we are searching identical page to.
|
|
* @rmap_item: the reverse mapping into the virtual address of this page
|
|
*/
|
|
static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
|
|
{
|
|
struct rmap_item *tree_rmap_item;
|
|
struct page *tree_page = NULL;
|
|
struct stable_node *stable_node;
|
|
struct page *kpage;
|
|
unsigned int checksum;
|
|
int err;
|
|
|
|
stable_node = page_stable_node(page);
|
|
if (stable_node) {
|
|
if (stable_node->head != &migrate_nodes &&
|
|
get_kpfn_nid(stable_node->kpfn) != NUMA(stable_node->nid)) {
|
|
rb_erase(&stable_node->node,
|
|
root_stable_tree + NUMA(stable_node->nid));
|
|
stable_node->head = &migrate_nodes;
|
|
list_add(&stable_node->list, stable_node->head);
|
|
}
|
|
if (stable_node->head != &migrate_nodes &&
|
|
rmap_item->head == stable_node)
|
|
return;
|
|
}
|
|
|
|
/* We first start with searching the page inside the stable tree */
|
|
kpage = stable_tree_search(page);
|
|
if (kpage == page && rmap_item->head == stable_node) {
|
|
put_page(kpage);
|
|
return;
|
|
}
|
|
|
|
remove_rmap_item_from_tree(rmap_item);
|
|
|
|
if (kpage) {
|
|
err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
|
|
if (!err) {
|
|
/*
|
|
* The page was successfully merged:
|
|
* add its rmap_item to the stable tree.
|
|
*/
|
|
lock_page(kpage);
|
|
stable_tree_append(rmap_item, page_stable_node(kpage));
|
|
unlock_page(kpage);
|
|
}
|
|
put_page(kpage);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* If the hash value of the page has changed from the last time
|
|
* we calculated it, this page is changing frequently: therefore we
|
|
* don't want to insert it in the unstable tree, and we don't want
|
|
* to waste our time searching for something identical to it there.
|
|
*/
|
|
checksum = calc_checksum(page);
|
|
if (rmap_item->oldchecksum != checksum) {
|
|
rmap_item->oldchecksum = checksum;
|
|
return;
|
|
}
|
|
|
|
tree_rmap_item =
|
|
unstable_tree_search_insert(rmap_item, page, &tree_page);
|
|
if (tree_rmap_item) {
|
|
kpage = try_to_merge_two_pages(rmap_item, page,
|
|
tree_rmap_item, tree_page);
|
|
put_page(tree_page);
|
|
if (kpage) {
|
|
/*
|
|
* The pages were successfully merged: insert new
|
|
* node in the stable tree and add both rmap_items.
|
|
*/
|
|
lock_page(kpage);
|
|
stable_node = stable_tree_insert(kpage);
|
|
if (stable_node) {
|
|
stable_tree_append(tree_rmap_item, stable_node);
|
|
stable_tree_append(rmap_item, stable_node);
|
|
}
|
|
unlock_page(kpage);
|
|
|
|
/*
|
|
* If we fail to insert the page into the stable tree,
|
|
* we will have 2 virtual addresses that are pointing
|
|
* to a ksm page left outside the stable tree,
|
|
* in which case we need to break_cow on both.
|
|
*/
|
|
if (!stable_node) {
|
|
break_cow(tree_rmap_item);
|
|
break_cow(rmap_item);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
|
|
struct rmap_item **rmap_list,
|
|
unsigned long addr)
|
|
{
|
|
struct rmap_item *rmap_item;
|
|
|
|
while (*rmap_list) {
|
|
rmap_item = *rmap_list;
|
|
if ((rmap_item->address & PAGE_MASK) == addr)
|
|
return rmap_item;
|
|
if (rmap_item->address > addr)
|
|
break;
|
|
*rmap_list = rmap_item->rmap_list;
|
|
remove_rmap_item_from_tree(rmap_item);
|
|
free_rmap_item(rmap_item);
|
|
}
|
|
|
|
rmap_item = alloc_rmap_item();
|
|
if (rmap_item) {
|
|
/* It has already been zeroed */
|
|
rmap_item->mm = mm_slot->mm;
|
|
rmap_item->address = addr;
|
|
rmap_item->rmap_list = *rmap_list;
|
|
*rmap_list = rmap_item;
|
|
}
|
|
return rmap_item;
|
|
}
|
|
|
|
static struct rmap_item *scan_get_next_rmap_item(struct page **page)
|
|
{
|
|
struct mm_struct *mm;
|
|
struct mm_slot *slot;
|
|
struct vm_area_struct *vma;
|
|
struct rmap_item *rmap_item;
|
|
int nid;
|
|
|
|
if (list_empty(&ksm_mm_head.mm_list))
|
|
return NULL;
|
|
|
|
slot = ksm_scan.mm_slot;
|
|
if (slot == &ksm_mm_head) {
|
|
/*
|
|
* A number of pages can hang around indefinitely on per-cpu
|
|
* pagevecs, raised page count preventing write_protect_page
|
|
* from merging them. Though it doesn't really matter much,
|
|
* it is puzzling to see some stuck in pages_volatile until
|
|
* other activity jostles them out, and they also prevented
|
|
* LTP's KSM test from succeeding deterministically; so drain
|
|
* them here (here rather than on entry to ksm_do_scan(),
|
|
* so we don't IPI too often when pages_to_scan is set low).
|
|
*/
|
|
lru_add_drain_all();
|
|
|
|
/*
|
|
* Whereas stale stable_nodes on the stable_tree itself
|
|
* get pruned in the regular course of stable_tree_search(),
|
|
* those moved out to the migrate_nodes list can accumulate:
|
|
* so prune them once before each full scan.
|
|
*/
|
|
if (!ksm_merge_across_nodes) {
|
|
struct stable_node *stable_node;
|
|
struct list_head *this, *next;
|
|
struct page *page;
|
|
|
|
list_for_each_safe(this, next, &migrate_nodes) {
|
|
stable_node = list_entry(this,
|
|
struct stable_node, list);
|
|
page = get_ksm_page(stable_node, false);
|
|
if (page)
|
|
put_page(page);
|
|
cond_resched();
|
|
}
|
|
}
|
|
|
|
for (nid = 0; nid < ksm_nr_node_ids; nid++)
|
|
root_unstable_tree[nid] = RB_ROOT;
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
|
|
ksm_scan.mm_slot = slot;
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
/*
|
|
* Although we tested list_empty() above, a racing __ksm_exit
|
|
* of the last mm on the list may have removed it since then.
|
|
*/
|
|
if (slot == &ksm_mm_head)
|
|
return NULL;
|
|
next_mm:
|
|
ksm_scan.address = 0;
|
|
ksm_scan.rmap_list = &slot->rmap_list;
|
|
}
|
|
|
|
mm = slot->mm;
|
|
down_read(&mm->mmap_sem);
|
|
if (ksm_test_exit(mm))
|
|
vma = NULL;
|
|
else
|
|
vma = find_vma(mm, ksm_scan.address);
|
|
|
|
for (; vma; vma = vma->vm_next) {
|
|
if (!(vma->vm_flags & VM_MERGEABLE))
|
|
continue;
|
|
if (ksm_scan.address < vma->vm_start)
|
|
ksm_scan.address = vma->vm_start;
|
|
if (!vma->anon_vma)
|
|
ksm_scan.address = vma->vm_end;
|
|
|
|
while (ksm_scan.address < vma->vm_end) {
|
|
if (ksm_test_exit(mm))
|
|
break;
|
|
*page = follow_page(vma, ksm_scan.address, FOLL_GET);
|
|
if (IS_ERR_OR_NULL(*page)) {
|
|
ksm_scan.address += PAGE_SIZE;
|
|
cond_resched();
|
|
continue;
|
|
}
|
|
if (PageAnon(*page) ||
|
|
page_trans_compound_anon(*page)) {
|
|
flush_anon_page(vma, *page, ksm_scan.address);
|
|
flush_dcache_page(*page);
|
|
rmap_item = get_next_rmap_item(slot,
|
|
ksm_scan.rmap_list, ksm_scan.address);
|
|
if (rmap_item) {
|
|
ksm_scan.rmap_list =
|
|
&rmap_item->rmap_list;
|
|
ksm_scan.address += PAGE_SIZE;
|
|
} else
|
|
put_page(*page);
|
|
up_read(&mm->mmap_sem);
|
|
return rmap_item;
|
|
}
|
|
put_page(*page);
|
|
ksm_scan.address += PAGE_SIZE;
|
|
cond_resched();
|
|
}
|
|
}
|
|
|
|
if (ksm_test_exit(mm)) {
|
|
ksm_scan.address = 0;
|
|
ksm_scan.rmap_list = &slot->rmap_list;
|
|
}
|
|
/*
|
|
* Nuke all the rmap_items that are above this current rmap:
|
|
* because there were no VM_MERGEABLE vmas with such addresses.
|
|
*/
|
|
remove_trailing_rmap_items(slot, ksm_scan.rmap_list);
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
ksm_scan.mm_slot = list_entry(slot->mm_list.next,
|
|
struct mm_slot, mm_list);
|
|
if (ksm_scan.address == 0) {
|
|
/*
|
|
* We've completed a full scan of all vmas, holding mmap_sem
|
|
* throughout, and found no VM_MERGEABLE: so do the same as
|
|
* __ksm_exit does to remove this mm from all our lists now.
|
|
* This applies either when cleaning up after __ksm_exit
|
|
* (but beware: we can reach here even before __ksm_exit),
|
|
* or when all VM_MERGEABLE areas have been unmapped (and
|
|
* mmap_sem then protects against race with MADV_MERGEABLE).
|
|
*/
|
|
hash_del(&slot->link);
|
|
list_del(&slot->mm_list);
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
|
|
free_mm_slot(slot);
|
|
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
|
|
up_read(&mm->mmap_sem);
|
|
mmdrop(mm);
|
|
} else {
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
up_read(&mm->mmap_sem);
|
|
}
|
|
|
|
/* Repeat until we've completed scanning the whole list */
|
|
slot = ksm_scan.mm_slot;
|
|
if (slot != &ksm_mm_head)
|
|
goto next_mm;
|
|
|
|
ksm_scan.seqnr++;
|
|
return NULL;
|
|
}
|
|
|
|
/**
|
|
* ksm_do_scan - the ksm scanner main worker function.
|
|
* @scan_npages - number of pages we want to scan before we return.
|
|
*/
|
|
static void ksm_do_scan(unsigned int scan_npages)
|
|
{
|
|
struct rmap_item *rmap_item;
|
|
struct page *uninitialized_var(page);
|
|
|
|
while (scan_npages-- && likely(!freezing(current))) {
|
|
cond_resched();
|
|
rmap_item = scan_get_next_rmap_item(&page);
|
|
if (!rmap_item)
|
|
return;
|
|
cmp_and_merge_page(page, rmap_item);
|
|
put_page(page);
|
|
}
|
|
}
|
|
|
|
static void process_timeout(unsigned long __data)
|
|
{
|
|
wake_up_process((struct task_struct *)__data);
|
|
}
|
|
|
|
static signed long __sched deferred_schedule_timeout(signed long timeout)
|
|
{
|
|
struct timer_list timer;
|
|
unsigned long expire;
|
|
|
|
__set_current_state(TASK_INTERRUPTIBLE);
|
|
if (timeout < 0) {
|
|
pr_err("schedule_timeout: wrong timeout value %lx\n",
|
|
timeout);
|
|
__set_current_state(TASK_RUNNING);
|
|
goto out;
|
|
}
|
|
|
|
expire = timeout + jiffies;
|
|
|
|
setup_deferrable_timer_on_stack(&timer, process_timeout,
|
|
(unsigned long)current);
|
|
mod_timer(&timer, expire);
|
|
schedule();
|
|
del_singleshot_timer_sync(&timer);
|
|
|
|
/* Remove the timer from the object tracker */
|
|
destroy_timer_on_stack(&timer);
|
|
|
|
timeout = expire - jiffies;
|
|
|
|
out:
|
|
return timeout < 0 ? 0 : timeout;
|
|
}
|
|
|
|
static int ksmd_should_run(void)
|
|
{
|
|
return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
|
|
}
|
|
|
|
static int ksm_scan_thread(void *nothing)
|
|
{
|
|
set_freezable();
|
|
set_user_nice(current, 5);
|
|
|
|
while (!kthread_should_stop()) {
|
|
mutex_lock(&ksm_thread_mutex);
|
|
wait_while_offlining();
|
|
if (ksmd_should_run())
|
|
ksm_do_scan(ksm_thread_pages_to_scan);
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
|
|
try_to_freeze();
|
|
|
|
if (ksmd_should_run()) {
|
|
if (use_deferred_timer)
|
|
deferred_schedule_timeout(
|
|
msecs_to_jiffies(ksm_thread_sleep_millisecs));
|
|
else
|
|
schedule_timeout_interruptible(
|
|
msecs_to_jiffies(ksm_thread_sleep_millisecs));
|
|
} else {
|
|
wait_event_freezable(ksm_thread_wait,
|
|
ksmd_should_run() || kthread_should_stop());
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end, int advice, unsigned long *vm_flags)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
int err;
|
|
|
|
switch (advice) {
|
|
case MADV_MERGEABLE:
|
|
/*
|
|
* Be somewhat over-protective for now!
|
|
*/
|
|
if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE |
|
|
VM_PFNMAP | VM_IO | VM_DONTEXPAND |
|
|
VM_HUGETLB | VM_MIXEDMAP))
|
|
return 0; /* just ignore the advice */
|
|
|
|
#ifdef VM_SAO
|
|
if (*vm_flags & VM_SAO)
|
|
return 0;
|
|
#endif
|
|
|
|
if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {
|
|
err = __ksm_enter(mm);
|
|
if (err)
|
|
return err;
|
|
}
|
|
|
|
*vm_flags |= VM_MERGEABLE;
|
|
break;
|
|
|
|
case MADV_UNMERGEABLE:
|
|
if (!(*vm_flags & VM_MERGEABLE))
|
|
return 0; /* just ignore the advice */
|
|
|
|
if (vma->anon_vma) {
|
|
err = unmerge_ksm_pages(vma, start, end);
|
|
if (err)
|
|
return err;
|
|
}
|
|
|
|
*vm_flags &= ~VM_MERGEABLE;
|
|
break;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int __ksm_enter(struct mm_struct *mm)
|
|
{
|
|
struct mm_slot *mm_slot;
|
|
int needs_wakeup;
|
|
|
|
mm_slot = alloc_mm_slot();
|
|
if (!mm_slot)
|
|
return -ENOMEM;
|
|
|
|
/* Check ksm_run too? Would need tighter locking */
|
|
needs_wakeup = list_empty(&ksm_mm_head.mm_list);
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
insert_to_mm_slots_hash(mm, mm_slot);
|
|
/*
|
|
* When KSM_RUN_MERGE (or KSM_RUN_STOP),
|
|
* insert just behind the scanning cursor, to let the area settle
|
|
* down a little; when fork is followed by immediate exec, we don't
|
|
* want ksmd to waste time setting up and tearing down an rmap_list.
|
|
*
|
|
* But when KSM_RUN_UNMERGE, it's important to insert ahead of its
|
|
* scanning cursor, otherwise KSM pages in newly forked mms will be
|
|
* missed: then we might as well insert at the end of the list.
|
|
*/
|
|
if (ksm_run & KSM_RUN_UNMERGE)
|
|
list_add_tail(&mm_slot->mm_list, &ksm_mm_head.mm_list);
|
|
else
|
|
list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
|
|
set_bit(MMF_VM_MERGEABLE, &mm->flags);
|
|
atomic_inc(&mm->mm_count);
|
|
|
|
if (needs_wakeup)
|
|
wake_up_interruptible(&ksm_thread_wait);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void __ksm_exit(struct mm_struct *mm)
|
|
{
|
|
struct mm_slot *mm_slot;
|
|
int easy_to_free = 0;
|
|
|
|
/*
|
|
* This process is exiting: if it's straightforward (as is the
|
|
* case when ksmd was never running), free mm_slot immediately.
|
|
* But if it's at the cursor or has rmap_items linked to it, use
|
|
* mmap_sem to synchronize with any break_cows before pagetables
|
|
* are freed, and leave the mm_slot on the list for ksmd to free.
|
|
* Beware: ksm may already have noticed it exiting and freed the slot.
|
|
*/
|
|
|
|
spin_lock(&ksm_mmlist_lock);
|
|
mm_slot = get_mm_slot(mm);
|
|
if (mm_slot && ksm_scan.mm_slot != mm_slot) {
|
|
if (!mm_slot->rmap_list) {
|
|
hash_del(&mm_slot->link);
|
|
list_del(&mm_slot->mm_list);
|
|
easy_to_free = 1;
|
|
} else {
|
|
list_move(&mm_slot->mm_list,
|
|
&ksm_scan.mm_slot->mm_list);
|
|
}
|
|
}
|
|
spin_unlock(&ksm_mmlist_lock);
|
|
|
|
if (easy_to_free) {
|
|
free_mm_slot(mm_slot);
|
|
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
|
|
mmdrop(mm);
|
|
} else if (mm_slot) {
|
|
down_write(&mm->mmap_sem);
|
|
up_write(&mm->mmap_sem);
|
|
}
|
|
}
|
|
|
|
struct page *ksm_might_need_to_copy(struct page *page,
|
|
struct vm_area_struct *vma, unsigned long address)
|
|
{
|
|
struct anon_vma *anon_vma = page_anon_vma(page);
|
|
struct page *new_page;
|
|
|
|
if (PageKsm(page)) {
|
|
if (page_stable_node(page) &&
|
|
!(ksm_run & KSM_RUN_UNMERGE))
|
|
return page; /* no need to copy it */
|
|
} else if (!anon_vma) {
|
|
return page; /* no need to copy it */
|
|
} else if (anon_vma->root == vma->anon_vma->root &&
|
|
page->index == linear_page_index(vma, address)) {
|
|
return page; /* still no need to copy it */
|
|
}
|
|
if (!PageUptodate(page))
|
|
return page; /* let do_swap_page report the error */
|
|
|
|
new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
|
|
if (new_page) {
|
|
copy_user_highpage(new_page, page, address, vma);
|
|
|
|
SetPageDirty(new_page);
|
|
__SetPageUptodate(new_page);
|
|
__set_page_locked(new_page);
|
|
}
|
|
|
|
return new_page;
|
|
}
|
|
|
|
int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
|
|
{
|
|
struct stable_node *stable_node;
|
|
struct rmap_item *rmap_item;
|
|
int ret = SWAP_AGAIN;
|
|
int search_new_forks = 0;
|
|
|
|
VM_BUG_ON_PAGE(!PageKsm(page), page);
|
|
|
|
/*
|
|
* Rely on the page lock to protect against concurrent modifications
|
|
* to that page's node of the stable tree.
|
|
*/
|
|
VM_BUG_ON_PAGE(!PageLocked(page), page);
|
|
|
|
stable_node = page_stable_node(page);
|
|
if (!stable_node)
|
|
return ret;
|
|
|
|
again:
|
|
hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
|
|
struct anon_vma *anon_vma = rmap_item->anon_vma;
|
|
struct anon_vma_chain *vmac;
|
|
struct vm_area_struct *vma;
|
|
|
|
cond_resched();
|
|
anon_vma_lock_read(anon_vma);
|
|
anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
|
|
0, ULONG_MAX) {
|
|
cond_resched();
|
|
vma = vmac->vma;
|
|
if (rmap_item->address < vma->vm_start ||
|
|
rmap_item->address >= vma->vm_end)
|
|
continue;
|
|
/*
|
|
* Initially we examine only the vma which covers this
|
|
* rmap_item; but later, if there is still work to do,
|
|
* we examine covering vmas in other mms: in case they
|
|
* were forked from the original since ksmd passed.
|
|
*/
|
|
if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
|
|
continue;
|
|
|
|
if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
|
|
continue;
|
|
|
|
ret = rwc->rmap_one(page, vma,
|
|
rmap_item->address, rwc->arg);
|
|
if (ret != SWAP_AGAIN) {
|
|
anon_vma_unlock_read(anon_vma);
|
|
goto out;
|
|
}
|
|
if (rwc->done && rwc->done(page)) {
|
|
anon_vma_unlock_read(anon_vma);
|
|
goto out;
|
|
}
|
|
}
|
|
anon_vma_unlock_read(anon_vma);
|
|
}
|
|
if (!search_new_forks++)
|
|
goto again;
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
#ifdef CONFIG_MIGRATION
|
|
void ksm_migrate_page(struct page *newpage, struct page *oldpage)
|
|
{
|
|
struct stable_node *stable_node;
|
|
|
|
VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
|
|
VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
|
|
VM_BUG_ON_PAGE(newpage->mapping != oldpage->mapping, newpage);
|
|
|
|
stable_node = page_stable_node(newpage);
|
|
if (stable_node) {
|
|
VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);
|
|
stable_node->kpfn = page_to_pfn(newpage);
|
|
/*
|
|
* newpage->mapping was set in advance; now we need smp_wmb()
|
|
* to make sure that the new stable_node->kpfn is visible
|
|
* to get_ksm_page() before it can see that oldpage->mapping
|
|
* has gone stale (or that PageSwapCache has been cleared).
|
|
*/
|
|
smp_wmb();
|
|
set_page_stable_node(oldpage, NULL);
|
|
}
|
|
}
|
|
#endif /* CONFIG_MIGRATION */
|
|
|
|
#ifdef CONFIG_MEMORY_HOTREMOVE
|
|
static void wait_while_offlining(void)
|
|
{
|
|
while (ksm_run & KSM_RUN_OFFLINE) {
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
wait_on_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE),
|
|
TASK_UNINTERRUPTIBLE);
|
|
mutex_lock(&ksm_thread_mutex);
|
|
}
|
|
}
|
|
|
|
static void ksm_check_stable_tree(unsigned long start_pfn,
|
|
unsigned long end_pfn)
|
|
{
|
|
struct stable_node *stable_node;
|
|
struct list_head *this, *next;
|
|
struct rb_node *node;
|
|
int nid;
|
|
|
|
for (nid = 0; nid < ksm_nr_node_ids; nid++) {
|
|
node = rb_first(root_stable_tree + nid);
|
|
while (node) {
|
|
stable_node = rb_entry(node, struct stable_node, node);
|
|
if (stable_node->kpfn >= start_pfn &&
|
|
stable_node->kpfn < end_pfn) {
|
|
/*
|
|
* Don't get_ksm_page, page has already gone:
|
|
* which is why we keep kpfn instead of page*
|
|
*/
|
|
remove_node_from_stable_tree(stable_node);
|
|
node = rb_first(root_stable_tree + nid);
|
|
} else
|
|
node = rb_next(node);
|
|
cond_resched();
|
|
}
|
|
}
|
|
list_for_each_safe(this, next, &migrate_nodes) {
|
|
stable_node = list_entry(this, struct stable_node, list);
|
|
if (stable_node->kpfn >= start_pfn &&
|
|
stable_node->kpfn < end_pfn)
|
|
remove_node_from_stable_tree(stable_node);
|
|
cond_resched();
|
|
}
|
|
}
|
|
|
|
static int ksm_memory_callback(struct notifier_block *self,
|
|
unsigned long action, void *arg)
|
|
{
|
|
struct memory_notify *mn = arg;
|
|
|
|
switch (action) {
|
|
case MEM_GOING_OFFLINE:
|
|
/*
|
|
* Prevent ksm_do_scan(), unmerge_and_remove_all_rmap_items()
|
|
* and remove_all_stable_nodes() while memory is going offline:
|
|
* it is unsafe for them to touch the stable tree at this time.
|
|
* But unmerge_ksm_pages(), rmap lookups and other entry points
|
|
* which do not need the ksm_thread_mutex are all safe.
|
|
*/
|
|
mutex_lock(&ksm_thread_mutex);
|
|
ksm_run |= KSM_RUN_OFFLINE;
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
break;
|
|
|
|
case MEM_OFFLINE:
|
|
/*
|
|
* Most of the work is done by page migration; but there might
|
|
* be a few stable_nodes left over, still pointing to struct
|
|
* pages which have been offlined: prune those from the tree,
|
|
* otherwise get_ksm_page() might later try to access a
|
|
* non-existent struct page.
|
|
*/
|
|
ksm_check_stable_tree(mn->start_pfn,
|
|
mn->start_pfn + mn->nr_pages);
|
|
/* fallthrough */
|
|
|
|
case MEM_CANCEL_OFFLINE:
|
|
mutex_lock(&ksm_thread_mutex);
|
|
ksm_run &= ~KSM_RUN_OFFLINE;
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
|
|
smp_mb(); /* wake_up_bit advises this */
|
|
wake_up_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE));
|
|
break;
|
|
}
|
|
return NOTIFY_OK;
|
|
}
|
|
#else
|
|
static void wait_while_offlining(void)
|
|
{
|
|
}
|
|
#endif /* CONFIG_MEMORY_HOTREMOVE */
|
|
|
|
#ifdef CONFIG_SYSFS
|
|
/*
|
|
* This all compiles without CONFIG_SYSFS, but is a waste of space.
|
|
*/
|
|
|
|
#define KSM_ATTR_RO(_name) \
|
|
static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
|
|
#define KSM_ATTR(_name) \
|
|
static struct kobj_attribute _name##_attr = \
|
|
__ATTR(_name, 0644, _name##_show, _name##_store)
|
|
|
|
static ssize_t sleep_millisecs_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%u\n", ksm_thread_sleep_millisecs);
|
|
}
|
|
|
|
static ssize_t sleep_millisecs_store(struct kobject *kobj,
|
|
struct kobj_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
unsigned long msecs;
|
|
int err;
|
|
|
|
err = kstrtoul(buf, 10, &msecs);
|
|
if (err || msecs > UINT_MAX)
|
|
return -EINVAL;
|
|
|
|
ksm_thread_sleep_millisecs = msecs;
|
|
|
|
return count;
|
|
}
|
|
KSM_ATTR(sleep_millisecs);
|
|
|
|
static ssize_t pages_to_scan_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%u\n", ksm_thread_pages_to_scan);
|
|
}
|
|
|
|
static ssize_t pages_to_scan_store(struct kobject *kobj,
|
|
struct kobj_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
int err;
|
|
unsigned long nr_pages;
|
|
|
|
err = kstrtoul(buf, 10, &nr_pages);
|
|
if (err || nr_pages > UINT_MAX)
|
|
return -EINVAL;
|
|
|
|
ksm_thread_pages_to_scan = nr_pages;
|
|
|
|
return count;
|
|
}
|
|
KSM_ATTR(pages_to_scan);
|
|
|
|
static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
|
|
char *buf)
|
|
{
|
|
return sprintf(buf, "%lu\n", ksm_run);
|
|
}
|
|
|
|
static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
int err;
|
|
unsigned long flags;
|
|
|
|
err = kstrtoul(buf, 10, &flags);
|
|
if (err || flags > UINT_MAX)
|
|
return -EINVAL;
|
|
if (flags > KSM_RUN_UNMERGE)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* KSM_RUN_MERGE sets ksmd running, and 0 stops it running.
|
|
* KSM_RUN_UNMERGE stops it running and unmerges all rmap_items,
|
|
* breaking COW to free the pages_shared (but leaves mm_slots
|
|
* on the list for when ksmd may be set running again).
|
|
*/
|
|
|
|
mutex_lock(&ksm_thread_mutex);
|
|
wait_while_offlining();
|
|
if (ksm_run != flags) {
|
|
ksm_run = flags;
|
|
if (flags & KSM_RUN_UNMERGE) {
|
|
set_current_oom_origin();
|
|
err = unmerge_and_remove_all_rmap_items();
|
|
clear_current_oom_origin();
|
|
if (err) {
|
|
ksm_run = KSM_RUN_STOP;
|
|
count = err;
|
|
}
|
|
}
|
|
}
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
|
|
if (flags & KSM_RUN_MERGE)
|
|
wake_up_interruptible(&ksm_thread_wait);
|
|
|
|
return count;
|
|
}
|
|
KSM_ATTR(run);
|
|
|
|
static ssize_t deferred_timer_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return snprintf(buf, 8, "%d\n", use_deferred_timer);
|
|
}
|
|
|
|
static ssize_t deferred_timer_store(struct kobject *kobj,
|
|
struct kobj_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
unsigned long enable;
|
|
int err;
|
|
|
|
err = kstrtoul(buf, 10, &enable);
|
|
use_deferred_timer = enable;
|
|
|
|
return count;
|
|
}
|
|
KSM_ATTR(deferred_timer);
|
|
|
|
#ifdef CONFIG_NUMA
|
|
static ssize_t merge_across_nodes_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%u\n", ksm_merge_across_nodes);
|
|
}
|
|
|
|
static ssize_t merge_across_nodes_store(struct kobject *kobj,
|
|
struct kobj_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
int err;
|
|
unsigned long knob;
|
|
|
|
err = kstrtoul(buf, 10, &knob);
|
|
if (err)
|
|
return err;
|
|
if (knob > 1)
|
|
return -EINVAL;
|
|
|
|
mutex_lock(&ksm_thread_mutex);
|
|
wait_while_offlining();
|
|
if (ksm_merge_across_nodes != knob) {
|
|
if (ksm_pages_shared || remove_all_stable_nodes())
|
|
err = -EBUSY;
|
|
else if (root_stable_tree == one_stable_tree) {
|
|
struct rb_root *buf;
|
|
/*
|
|
* This is the first time that we switch away from the
|
|
* default of merging across nodes: must now allocate
|
|
* a buffer to hold as many roots as may be needed.
|
|
* Allocate stable and unstable together:
|
|
* MAXSMP NODES_SHIFT 10 will use 16kB.
|
|
*/
|
|
buf = kcalloc(nr_node_ids + nr_node_ids, sizeof(*buf),
|
|
GFP_KERNEL);
|
|
/* Let us assume that RB_ROOT is NULL is zero */
|
|
if (!buf)
|
|
err = -ENOMEM;
|
|
else {
|
|
root_stable_tree = buf;
|
|
root_unstable_tree = buf + nr_node_ids;
|
|
/* Stable tree is empty but not the unstable */
|
|
root_unstable_tree[0] = one_unstable_tree[0];
|
|
}
|
|
}
|
|
if (!err) {
|
|
ksm_merge_across_nodes = knob;
|
|
ksm_nr_node_ids = knob ? 1 : nr_node_ids;
|
|
}
|
|
}
|
|
mutex_unlock(&ksm_thread_mutex);
|
|
|
|
return err ? err : count;
|
|
}
|
|
KSM_ATTR(merge_across_nodes);
|
|
#endif
|
|
|
|
static ssize_t pages_shared_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%lu\n", ksm_pages_shared);
|
|
}
|
|
KSM_ATTR_RO(pages_shared);
|
|
|
|
static ssize_t pages_sharing_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%lu\n", ksm_pages_sharing);
|
|
}
|
|
KSM_ATTR_RO(pages_sharing);
|
|
|
|
static ssize_t pages_unshared_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%lu\n", ksm_pages_unshared);
|
|
}
|
|
KSM_ATTR_RO(pages_unshared);
|
|
|
|
static ssize_t pages_volatile_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
long ksm_pages_volatile;
|
|
|
|
ksm_pages_volatile = ksm_rmap_items - ksm_pages_shared
|
|
- ksm_pages_sharing - ksm_pages_unshared;
|
|
/*
|
|
* It was not worth any locking to calculate that statistic,
|
|
* but it might therefore sometimes be negative: conceal that.
|
|
*/
|
|
if (ksm_pages_volatile < 0)
|
|
ksm_pages_volatile = 0;
|
|
return sprintf(buf, "%ld\n", ksm_pages_volatile);
|
|
}
|
|
KSM_ATTR_RO(pages_volatile);
|
|
|
|
static ssize_t full_scans_show(struct kobject *kobj,
|
|
struct kobj_attribute *attr, char *buf)
|
|
{
|
|
return sprintf(buf, "%lu\n", ksm_scan.seqnr);
|
|
}
|
|
KSM_ATTR_RO(full_scans);
|
|
|
|
static struct attribute *ksm_attrs[] = {
|
|
&sleep_millisecs_attr.attr,
|
|
&pages_to_scan_attr.attr,
|
|
&run_attr.attr,
|
|
&pages_shared_attr.attr,
|
|
&pages_sharing_attr.attr,
|
|
&pages_unshared_attr.attr,
|
|
&pages_volatile_attr.attr,
|
|
&full_scans_attr.attr,
|
|
&deferred_timer_attr.attr,
|
|
#ifdef CONFIG_NUMA
|
|
&merge_across_nodes_attr.attr,
|
|
#endif
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute_group ksm_attr_group = {
|
|
.attrs = ksm_attrs,
|
|
.name = "ksm",
|
|
};
|
|
#endif /* CONFIG_SYSFS */
|
|
|
|
static int __init ksm_init(void)
|
|
{
|
|
struct task_struct *ksm_thread;
|
|
int err;
|
|
|
|
err = ksm_slab_init();
|
|
if (err)
|
|
goto out;
|
|
|
|
ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
|
|
if (IS_ERR(ksm_thread)) {
|
|
pr_err("ksm: creating kthread failed\n");
|
|
err = PTR_ERR(ksm_thread);
|
|
goto out_free;
|
|
}
|
|
|
|
#ifdef CONFIG_SYSFS
|
|
err = sysfs_create_group(mm_kobj, &ksm_attr_group);
|
|
if (err) {
|
|
pr_err("ksm: register sysfs failed\n");
|
|
kthread_stop(ksm_thread);
|
|
goto out_free;
|
|
}
|
|
#else
|
|
ksm_run = KSM_RUN_MERGE; /* no way for user to start it */
|
|
|
|
#endif /* CONFIG_SYSFS */
|
|
|
|
#ifdef CONFIG_MEMORY_HOTREMOVE
|
|
/* There is no significance to this priority 100 */
|
|
hotplug_memory_notifier(ksm_memory_callback, 100);
|
|
#endif
|
|
|
|
show_mem_notifier_register(&ksm_show_mem_notifier_block);
|
|
return 0;
|
|
|
|
out_free:
|
|
ksm_slab_free();
|
|
out:
|
|
return err;
|
|
}
|
|
subsys_initcall(ksm_init);
|