================================================================================ ==================================== --Network Configuration for 11g (11.2.0.2) RAC with grid Infrastructure ================================================================================ ==================================== vi /etc/sysctl.

conf added net.ipv4.conf.all.rp_filter = 1 to enable strong host model in which packe d sending and receivign is based on seding and receiving interface name

-simplay chaning by mac address you can change the interface name change the mac address of eth1 to go though the lom1-a which is configured for c luster VLAN /etc/sysconfig/network-scripts/ifcfg-eth1 change the HWADDR parameter as per you r choice #HWADDR=98:4B:E1:08:55:40 #HWADDR=98:4b:e1:08:55:34 HWADDR=98:4B:E1:08:55:40 ================================================================================ ==================================== --Storage Configuration for 11g (11.2.0.2) RAC with grid Infrastructure (no need configure udev rules--set permissions though start script and naming wi ht multipath configuration file) ================================================================================ ==================================== configure multipath first multipath -ll /sbin/chkconfig multipathd on /sbin/chkconfig --add multipathd /etc/init.d/multipathd status /etc/init.d/multipathd start vi /etc/multipath.conf devices { device { vendor "NETAPP" product "LUN" path_grouping_policy group_by_prio features "1 queue_if_no_path" prio_callout "/sbin/mpath_prio_alua /dev/%n" path_checker directio failback immediate hardware_handler "1 alua" rr_weight uniform rr_min_io 128 getuid_callout "/sbin/scsi_id -g -u -s /block/%n" } } devnode_blacklist {

devnode "^(ram raw loop fd md dm- sr scd st)[0-9]*" devnode "^hd[a-z]" } multipaths { multipath { wwid 360a98000572d4f395a6f677058596c49 alias dg_ocr01 path_grouping_policy failover } multipath { wwid 360a98000572d4f395a6f677058625a52 alias dg_data01 path_grouping_policy failover } multipath { wwid 360a98000572d4f395a6f677058654e4a alias dg_data02 path_grouping_policy failover } }

WWN cat /sys/class/scsi_host/host0/device/fc_host:host0/port_name cat /sys/class/scsi_host/host1/device/fc_host:host1/port_name send WWN to storage administrator so that he can present SAN LUN to it echo "- - -" > /sys/class/scsi_host/host0/scan echo "- - -" > /sys/class/scsi_host/host1/scan fidks -l will list the LUN after scsi bus has been scanned multipath -ll for i in `cat /proc/partitions awk {'print $4'} grep sd`; do echo "### $i: ` scsi_id -g -u -s /block/$i`"; done The /dev/dm-N devices are used internally by device-mapper-multipath and are non -persistent across reboot, so should not be used. The /dev/mpath/ devices are created for multipath devices to be visible together , however, may not be available during early stages of boot, so, again, should not be used. However, /dev/mapper/ devices are persistent, created sufficiently early during the boot process and makes use of their defined aliases use only these devices to access and interact with multipathed devices. # # # # # dmsetup ls sort ls -l /dev/disk/by-id/ ls -l /dev/mapper/ ls -l /dev/mpath/* ls -l /dev/dm-*

# cat /etc/rc.local #!/bin/sh

# # This script will be executed *after* all the other init scripts. # You can put your own initialization stuff in here if you don't # want to do the full Sys V style init stuff. touch /var/lock/subsys/local # GRID AND RAC disks 11gR2 chown grid:asmadmin /dev/mapper/dg_* chmod 0660 /dev/mapper/dg_* Re: 2008.1 - /etc/rc.local does not run at boot Aragorn: > You could simply edit that file and make it call /rc.local/ instead. :-) So I did (rc.early.local, if exists, is loaded _early_ that is not my case). Adding the following lines at the _end_ of /etc/rc.d/rc.sysinit did the job. if [ -f /etc/rc.local ]; then . /etc/rc.local fi ================================================================================ ==================================== --Time Configuration for 11g (11.2.0.2) RAC with grid Infrastructure ================================================================================ ====================================

/sbin/service ntpd status

./crsctl stop crs -f ./crsctl start cluster [root@retaildb01 app]# cd /u01/app rm -rf * cd /etc/oracle rm -rf * pkill -U grid remove +ASM1 entry from root@retaildb01 oracle]# vi /etc/oratab reboot [root@retaildb02 app]# cd /u01/app rm -rf * cd /etc/oracle rm -rf * pkill -U grid

remove +ASM2 entry from root@retaildb02 oracle]# vi /etc/oratab reboot [root@retaildb01 app]# mkdir -p /u01/app/11.2.0/grid mkdir -p /u01/app/grid mkdir -p /u01/app/oracle chown grid:oinstall /u01/app/11.2.0/grid chown grid:oinstall /u01/app/grid chown oracle:oinstall /u01/app/oracle chmod -R 775 /u01/ chown -R grid:oinstall /u01

[root@retaildb02 app]# mkdir -p /u01/app/11.2.0/grid mkdir -p /u01/app/grid mkdir -p /u01/app/oracle chown grid:oinstall /u01/app/11.2.0/grid chown grid:oinstall /u01/app/grid chown oracle:oinstall /u01/app/oracle chmod -R 775 /u01/ chown -R grid:oinstall /u01 cd /dev/disk/by-id grep MemTotal /proc/meminfo grep SwapTotal /proc/meminfo df -h /tmp df h rpm -Uvh sysstat-7.0.2-3.el5.x86_64.rpm rpm -Uvh unixODBC-2.2.11-7.1.i386.rpm rpm -Uvh unixODBC-2.2.11-7.1.x86_64.rpm rpm -Uvh unixODBC-devel-2.2.11-7.1.i386.rpm rpm -Uvh unixODBC-devel-2.2.11-7.1.x86_64.rpm rpm -Uvh kernel-debug-2.6.18-194.el5.x86_64.rpm rpm -Uvh oracleasm-support-2.1.3-1.el5.x86_64.rpm rpm -Uvh oracleasm-2.6.18-194.el5-2.0.5-1.el5.x86_64.rpm rpm -Uvh oracleasm-2.6.18-194.el5debug-2.0.5-1.el5.x86_64.rpm rpm -Uvh libaio-devel-0.3.106-5.x86_64.rpm rpm -Uvh libaio-devel-0.3.106-5.i386.rpm

export CVUQDISK_GRP=oinstall rpm -Uvh cvuqdisk-1.0.7-1.rpm

pkill -U grid chown grid:asmadmin * ./mcasttest.pl -n retaildb01,retaildb02 -i eth0,eth1

./crsctl status resource -t crsctl start crs crsctl stop crs crsctl check crs

cd /u01/app/11.2.0/grid/bin [root@retaildb01 bin]# ./srvctl [root@retaildb01 bin]# ./srvctl [root@retaildb01 bin]# ./crsctl [root@retaildb01 bin]# ./srvctl [root@retaildb01 bin]# ./srvctl srvctl start asm -n <node name> srvctl stop asm -n <node name> crsctl query css votedisk

stop database -d retaildb stop database -d retaildb status resource ora.retaildb.db stop instance -d retaildb -i retaildb1 stop instance -d retaildb -i retaildb2

To prevent a database starting at boot time srvctl disable database -d <database name>

Retrive OCR (Oracle Cluster Registry) information ocrcheck

ALTER ALTER ALTER ALTER

SYSTEM SYSTEM SYSTEM SYSTEM

SET OPEN_CURSORS=2000 sid='*' SCOPE=MEMORY; RESET OPEN_CURSORS SCOPE=MEMORY sid='prod1'; RESET OPEN_CURSORS SCOPE=SPFILE sid='prod1'; RESET OPEN_CURSORS SCOPE=SPFILE sid='*';

[root@retaildb01 bin]# pwd /u01/app/11.2.0/grid/bin [root@retaildb01 bin]# ./crsctl check crs

select * from gv$instance

$ export ORACLE_SID=+ASM2 $ amdcmd ASMCMD> ls failover testing First Node ./crsctl status resource -t shutdown second node and run above coomand again to see if every thing failed ov er to first one or no also check alert log of first node. ================================================================================ ==================================== --Increase /dev/shm ================================================================================ ==================================== > umount tmpfs > mount -t tmpfs shmfs -o size=1500m /dev/shm SQL> startup ORA-00845: MEMORY_TARGET not supported on this system /etc/fstab change the size of tmpfs from 16g (default when total memory is 32g) to 24g old: 0 0 new: 24g tmpfs 0 0 /dev/shm tmpfs defaults,size= tmpfs /dev/shm tmpfs defaults

[root@whjed-bi03-db ~]# mount -o remount /dev/shm ab [root@whjed-bi03-db ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/mapper/VolGroup00-LogVol00 256G 176G 67G 73% / /dev/cciss/c0d0p1 99M 27M 67M 29% /boot tmpfs 24G 0 24G 0% /dev/shm /dev/dm-3 3.0T 1.3T 1.6T 44% /u02 [oracle@whjed-bi03-db scripts]$ sqlplus "/as sysdba"

? To re-read fst

SQL*Plus: Release 11.2.0.1.0 Production on Tue Nov 15 00:11:07 2011 Copyright (c) 1982, 2009, Oracle. All rights reserved. Connected to an idle instance.

For example, if you have 8GB RAM then remount /dev/shm as follows: # mount -o remount,size=8G /dev/shm To be frank, if you have more than 2GB RAM + multiple Virtual machines, this hac k always improves performance. # mount -t tmpfs -o size=5G,nr_inodes=5k,mode=700 tmpfs /disk2/tmpfs

================================================================================ =================================== --ASM commands ================================================================================ ==================================== ################################################################ # Adding/Removing/Managing configuration about existing ASM instances ################################################################ --Use the following syntax to add configuration information about an existing AS M instance: srvctl add asm -n node_name -i +asm_instance_name -o oracle_home --Use the following syntax to remove an ASM instance: srvctl remove asm -n node_name [-i +asm_instance_name] --Use the following syntax to enable an ASM instance: srvctl enable asm -n node_name [-i ] +asm_instance_name --Use the following syntax to disable an ASM instance: srvctl disable asm -n node_name [-i +asm_instance_name] --Use the following syntax to start an ASM instance: srvctl start asm -n node_name [-i +asm_instance_name] [-o start_options] --Use the following syntax to stop an ASM instance: srvctl stop asm -n node_name [-i +asm_instance_name] [-o stop_options] --Use the following syntax to show the configuration of an ASM instance: srvctl config asm -n node_name --Use the following syntax to obtain the status of an ASM instance: srvctl status asm -n node_name --P.S.: --For all of the SRVCTL commands in this section for which the --option is not required, if you do not specify an instance name, then -i --the command applies to all of the ASM instances on the node. ################################### # Managing DiskGroup inside ASM: ################################### --Note that adding or dropping disks will initiate a rebalance of the data on th e disks. --The status of these processes can be shown by selecting from v$asm_operation.

--Quering ASM Disk Groups col name format a25 col DATABASE_COMPATIBILITY format a10 col COMPATIBILITY format a10 select * from v$asm_diskgroup; --or select name, state, type, total_mb, free_mb from v$asm_diskgroup; --Quering ASM Disks col PATH format a55 col name format a25 select name, path, group_number, TOTAL_MB, FREE_MB, READS, WRITES, READ_TIME, WRITE_TIME from v$asm_disk order by 3,1; --or col PATH format a50 col HEADER_STATUS format a12 col name format a25 --select INCARNATION, select name, path, MOUNT_STATUS,HEADER_STATUS, MODE_STATUS, STATE, group_number, OS_MB, TOTAL_MB, FREE_MB, READS, WRITES, READ_TIME, WRITE_TIME, BYTES_READ, BYTES_WRITTEN, REPAIR_TIMER, MOUNT_DATE, CREATE_DATE from v$asm_disk; ################################### #TUNING and Analisys ################################### --Only Performance Statistics --N.B Time in Hundredseconds! col READ_TIME format 9999999999.99 col WRITE_TIME format 9999999999.99 col BYTES_READ format 99999999999999.99 col BYTES_WRITTEN format 99999999999999.99 select name, STATE, group_number, TOTAL_MB, FREE_MB,READS, WRITES, READ_TIME, WRITE_TIME, BYTES_READ, BYTES_WRITTEN, REPAIR_TIMER,MOUNT_DATE from v$asm_disk order by group_number, name; --Check the Num of Extents in use per Disk inside one Disk Group. select max(substr(name,1,30)) group_name, count(PXN_KFFXP) extents_per_disk, DISK_KFFXP, GROUP_KFFXP from x$kffxp, v$ASM_DISKGROUP gr where GROUP_KFFXP=&group_nr and GROUP_KFFXP=GROUP_NUMBER group by GROUP_KFFXP, DISK_KFFXP order by GROUP_KFFXP, DISK_KFFXP; --Find The File distribution Between Disks SELECT * FROM v$asm_alias WHERE name='PWX_DATA.272.669293645'; SELECT GROUP_KFFXP Group#,DISK_KFFXP Disk#,AU_KFFXP AU#,XNUM_KFFXP Extent# FROM X$KFFXP WHERE number_kffxp=(SELECT file_number FROM v$asm_alias WHERE name='PWX_DATA.272.669293645'); --or SELECT GROUP_KFFXP Group#,DISK_KFFXP Disk#,AU_KFFXP AU#,XNUM_KFFXP Extent# FROM X$KFFXP WHERE number_kffxp=&DataFile_Number; --or select d.name, XV.GROUP_KFFXP Group#, XV.DISK_KFFXP Disk#, XV.NUMBER_KFFXP File_Number, XV.AU_KFFXP AU#, XV.XNUM_KFFXP Extent#, XV.ADDR, XV.INDX, XV.INST_ID, XV.COMPOUND_KFFXP, XV.INCARN_KFFXP,

XV.PXN_KFFXP, XV.XNUM_KFFXP,XV.LXN_KFFXP, XV.FLAGS_KFFXP, XV.CHK_KFFXP, XV.SIZE_KFFXP from v$asm_disk d, X$KFFXP XV where d.GROUP_NUMBER=XV.GROUP_KFFXP and d.DISK_NUMBER=XV.DISK_KFFXP and number_kffxp=&File_NUM order by 2,3,4; --List the hierarchical tree of files stored in the diskgroup SELECT concat('+' gname, sys_connect_by_path(aname, '/')) full_alias_path FROM (SELECT g.name gname, a.parent_index pindex, a.name aname, a.reference_index rindex FROM v$asm_alias a, v$asm_diskgroup g WHERE a.group_number = g.group_number) START WITH (mod(pindex, power(2, 24))) = 0 CONNECT BY PRIOR rindex = pindex; ################################################################ ################################### #Create and Modify Disk Group ################################### create diskgroup FRA1 external redundancy disk '/dev/vx/rdsk/oraASMdg/fra1' ATTRIBUTE 'compatible.rdbms' = '11.1', 'compatible.asm' = '11.1'; alter diskgroup FRA1 check all; --on +ASM2 : alter diskgroup FRA1 mount; --Add a second disk: alter diskgroup FRA1 add disk '/dev/vx/rdsk/oraASMdg/fra2'; --Add several disks with a wildcard: alter diskgroup FRA1 add disk '/dev/vx/rdsk/oraASMdg/fra*'; --Remove a disk from a diskgroup: alter diskgroup FRA1 drop disk 'FRA1_0002'; --Drop the entire DiskGroup drop diskgroup DATA1 including contents; --How to DROP the entire DiskGroup when it is in NOMOUNT Status --Generate the dd command which will reset the header of all the --disks belong the GROUP_NUMBER=0!!!! select 'dd if=/dev/zero of=''' PATH ''' bs=8192 count=100' from v$asm_disk where GROUP_NUMBER=0; select * from v$asm_operation; -------------------------------------------------------------------------------------alter diskgroup FRA1 drop disk 'FRA1_0002'; alter diskgroup FRA1 add disk '/dev/vx/rdsk/fra1dg/fra3'; alter diskgroup FRA1 drop disk 'FRA1_0003';

alter diskgroup FRA1 add disk '/dev/vx/rdsk/fra1dg/fra4'; --When a new diskgroup is created, it is only mounted on the local instance, --and only the instance-specific entry for the asm_diskgroups parameter is updat ed. --By manually mounting the diskgroup on other instances, the asm_diskgroups para meter --on those instances are updated. --on +ASM1 : create diskgroup FRA1 external redundancy disk '/dev/vx/rdsk/fradg/fra1' ATTRIBUTE 'compatible.rdbms' = '11.1', 'compatible.asm' = '11.1'; --on +ASM2 : alter diskgroup FRA1 mount; --It works even for on going balances!!! alter diskgroup DATA1 rebalance power 10; ################################################################ # New ASM Command Line Utility (ASMCMD) Commands and Options ################################################################ ASMCMD Command Reference: Command Description -------------------- cd Command Changes the current directory to the specified directory. - cp Command Enables you to copy files between ASM disk groups on a local instan ce and remote instances. - du Command Displays the total disk space occupied by ASM files in the specifie d - ASM directory and all of its subdirectories, recursively. - exit Command Exits ASMCMD. - find Command Lists the paths of all occurrences of the specified name (with wi ldcards) under the specified directory. - help Command Displays the syntax and description of ASMCMD commands. - ls Command Lists the contents of an ASM directory, the attributes of the speci fied - file, or the names and attributes of all disk groups. - lsct Command Lists information about current ASM clients. - lsdg Command Lists all disk groups and their attributes. - lsdsk Command Lists disks visible to ASM. - md_backup Command Creates a backup of all of the mounted disk groups. - md_restore Command Restores disk groups from a backup. - mkalias Command Creates an alias for system-generated filenames. - mkdir Command Creates ASM directories. - pwd Command Displays the path of the current ASM directory. - remap Command Repairs a range of physical blocks on a disk. - rm Command Deletes the specified ASM files or directories. - rmalias Command Deletes the specified alias, retaining the file that the alias points to. --------- kfed tool From Unix Prompt for reading ASM disk header. kfed read /dev/vx/rdsk/fra1dg/fra1

################################################################ # CREATE and Manage Tablespaces and Datafiles on ASM ################################################################ CREATE TABLESPACE my_ts DATAFILE '+disk_group_1' SIZE 100M AUTOEXTEND ON; ALTER TABLESPACE sysaux ADD DATAFILE '+disk_group_1' SIZE 100M; ALTER DATABASE DATAFILE '+DATA1/dbname/datafile/audit.259.668957419' RESIZE 150M ; ------------------------create diskgroup DATA1 external redundancy disk '/dev/vx/rdsk/oraASMdg/fra1' ATTRIBUTE 'compatible.rdbms' = '11.1', 'compatible.asm' = '11.1'; select 'alter diskgroup DATA1 add disk ''' PATH where GROUP_NUMBER=0 and rownum<=&Num_Disks_to_add; ''';' from v$asm_disk

select 'alter diskgroup FRA1 add disk ''' PATH ''';' from v$asm_disk where GROUP_NUMBER=0 and rownum<=&Num_Disks_to_add; --Remove ASM header select 'dd if=/dev/zero of=''' where GROUP_NUMBER=0; libXm.so.2 => not found libXp.so.6 => not found libXm.so.2 => /usr/X11R6/lib/libXm.so.2 (0xf5b39000) libXt.so.6 => /usr/lib/libXt.so.6 (0x00688000)

PATH ''' bs=8192 count=100' from v$asm_disk